00000020:00000001:0.0F:1788572629.313960:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572629.313971:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572629.313974:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572630.144906:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572630.144916:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572630.144922:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572630.144932:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572630.144938:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572630.144948:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572630.144958:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572631.168949:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572631.168969:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572631.168981:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572631.168994:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572631.168998:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572631.169005:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572631.169010:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572632.193056:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572632.193068:0:2399:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1788572632.193085:0:2399:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788572616/real 1788572616] req@ffff8fc4b80c3b80 x1875440516472192/t0(0) o400->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572632 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572632.193176:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572632.193179:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572632.193194:0:2399:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915840. 00000400:00000200:0.0:1788572632.193201:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9330 00000400:00000010:0.0:1788572632.193209:0:2399:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9330. 00000100:00000001:0.0:1788572632.193224:0:2399:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572632.193239:0:2399:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c3b80 x1875440516472192/t0(0) o400->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572632 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572632.193314:0:2399:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c3b80 x1875440516472192/t0(0) o400->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572632 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572632.193383:0:2399:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572632.193388:0:2399:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572632.193399:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572632.193407:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572632.193420:0:2399:0:(client.c:2555:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8fc4b80c3b80 x1875440516472192/t0(0) o400->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572632 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572632.193461:0:2399:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1788572632.193468:0:2399:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8fc4912d1000 already not connected (conn 7, was 6): CONNECTING 00000100:00000001:0.0:1788572632.193486:0:2399:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1788572632.193491:0:2399:0:(client.c:2563:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572632.193509:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572632.193516:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=65 : 65 : 41) 00000100:00000001:0.0:1788572632.193524:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572632.193530:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572632.193536:0:2399:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572632.193545:0:2399:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572632.193554:0:2399:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572632.193571:0:2399:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c3b80 x1875440516472192/t0(0) o400->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572632 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572632.193611:0:2399:0:(client.c:2198:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:0.0:1788572632.193628:0:2399:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572632.193634:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572632.193637:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572632.193648:0:2399:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c3b80 x1875440516472192/t0(0) o400->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572632 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572632.193671:0:2399:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:51d6aedc-ea7c-426a-b495-22e82c13ee75:2399:1875440516472192:192.168.202.128@tcp:400:kworker.0 00000100:00000001:0.0:1788572632.193688:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572632.193699:0:2399:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572632.193711:0:2399:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c3b80 x1875440516472192/t0(0) o400->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572632 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572632.193730:0:2399:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572632.193735:0:2399:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572632.193746:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8fc4a1d6e000. 02000000:00000001:0.0:1788572632.193753:0:2399:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572632.193760:0:2399:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572632.193770:0:2399:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788572632.193782:0:2399:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572632.193785:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8fc39b5d0e00. 02000000:00000001:0.0:1788572632.193792:0:2399:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572632.193796:0:2399:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572632.193799:0:2399:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c3b80. 00000100:00000001:0.0:1788572632.193804:0:2399:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572632.193805:0:2399:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572632.193848:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572632.194927:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572632.194937:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572632.194943:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572632.194955:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5f80 x1875440516474624/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572632.195008:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572632.195016:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572632.195026:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572632.195035:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572632.195052:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572632.195067:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516474624:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788572632.195075:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572632.195082:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572632.195086:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572632.195096:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572632.195107:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc485f22000. 02000000:00000001:0.0:1788572632.195126:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572632.195151:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572632.195162:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915840. 00000400:00000010:0.0:1788572632.195174:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9330. 00000100:00000200:0.0:1788572632.195184:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516474624, portal 4 00000100:00000001:0.0:1788572632.195195:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572632.195200:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126850432 : -123400582701184 : ffff8fc4901a5f80) 00000100:00000040:0.0:1788572632.195220:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5f80 x1875440516474624/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572632.195288:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572632.195312:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572632.195327:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788572632.195342:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516474624, offset 0 00000400:00000200:0.0:1788572632.195360:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572632.195382:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572632.195404:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572632.195423:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572632.195439:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572632.195447:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572632.195462:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572632.195480:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545365:545365:256:4294967295] 192.168.202.128@tcp LPNI seq info [545288:545288:8:4294967295] 00000400:00000200:0.0:1788572632.195497:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572632.195534:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572632.195555:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572632.195568:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56300. 00000800:00000200:0.0:1788572632.195583:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (5) 00000800:00000200:0.0:1788572632.195602:0:2398:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.128@tcp ip 192.168.202.128:988 00000800:00000200:0.0:1788572632.195611:0:2398:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4b3b56300 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1788572632.195628:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572632.195636:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788572632.195646:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572632.195651:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4307992231 00000100:00000001:0.0:1788572632.195659:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4307995886 00000100:00000001:0.0:1788572632.195669:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4307992231 00000100:00000001:0.0:1788572632.195674:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4307995886 00000100:00000001:0.0:1788572632.195690:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572632.195695:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572632.195708:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572632.195714:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572632.195730:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572632.195740:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572632.195747:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572632.195753:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572632.195763:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572632.195773:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7b80 x1875440516474752/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572632.195877:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572632.195884:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572632.195890:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572632.195899:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572632.195920:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572632.195943:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516474752:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788572632.195960:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572632.195967:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572632.195971:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572632.195975:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572632.195979:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc485f20c00. 02000000:00000001:0.0:1788572632.195985:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572632.196002:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572632.196009:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915160. 00000400:00000010:0.0:1788572632.196016:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed96e8. 00000100:00000200:0.0:1788572632.196021:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516474752, portal 4 00000100:00000001:0.0:1788572632.196024:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572632.196028:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126857600 : -123400582694016 : ffff8fc4901a7b80) 00000100:00000040:0.0:1788572632.196047:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7b80 x1875440516474752/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572632.196056:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572632.196060:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572632.196063:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788572632.196065:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516474752, offset 0 00000400:00000200:0.0:1788572632.196071:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572632.196075:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572632.196079:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572632.196088:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572632.196090:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572632.196092:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572632.196094:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572632.196097:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545367:545367:256:4294967295] 192.168.202.128@tcp LPNI seq info [545290:545290:8:4294967295] 00000400:00000200:0.0:1788572632.196100:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572632.196106:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572632.196109:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572632.196124:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788572632.196128:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (5) 00000800:00000200:0.0:1788572632.196132:0:2398:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.128@tcp ip 192.168.202.128:988 00000800:00000200:0.0:1788572632.196134:0:2398:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4b3b56900 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1788572632.196144:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572632.196147:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788572632.196150:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572632.196152:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4307992234 00000100:00000001:0.0:1788572632.196154:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4307995886 00000100:00000001:0.0:1788572632.196156:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4307992234 00000100:00000001:0.0:1788572632.196158:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4307995886 00000100:00000001:0.0:1788572632.196160:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572632.196162:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572632.196166:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572632.196168:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572632.196171:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572632.196173:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572632.196174:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572632.196175:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572632.196180:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572632.196181:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572632.705023:0:242485:0:(rw.c:1480:ll_writepages()) Process entered 00000080:00000001:0.0:1788572632.705029:0:242485:0:(file.c:5295:cl_sync_file_range()) Process entered 00000020:00000001:0.0:1788572632.705053:0:242485:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572632.705058:0:242485:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788572632.705065:0:242485:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000020:00000001:0.0:1788572632.705073:0:242485:0:(cl_object.c:143:cl_object_top()) ffff8fc4accf8f58 -> ffff8fc4accf8f58 00000020:00000001:0.0:1788572632.705077:0:242485:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572632.705081:0:242485:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572632.705084:0:242485:0:(vvp_io.c:1826:vvp_io_init()) [0x2400013a6:0x10fc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572632.705089:0:242485:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572632.705091:0:242485:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572632.705093:0:242485:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572632.705111:0:242485:0:(lov_object.c:1590:lov_io_init()) [0x2400013a6:0x10fc:0x0]io ffff8fc487d8f590 type 6 ignore/verify layout 1/0 00020000:00000002:0.0:1788572632.705116:0:242485:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff8fc485c852e8) owner 0000000000000000/ffff8fc4a0a28000 00020000:00000001:0.0:1788572632.705122:0:242485:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572632.705124:0:242485:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572632.705127:0:242485:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572632.705129:0:242485:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705131:0:242485:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572632.705134:0:242485:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572632.705136:0:242485:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705139:0:242485:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572632.705140:0:242485:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572632.705142:0:242485:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572632.705144:0:242485:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff8fc485c852e8) owner 0000000000000000/ffff8fc4a0a28000 00000020:00000001:0.0:1788572632.705146:0:242485:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705149:0:242485:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1788572632.705150:0:242485:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1788572632.705153:0:242485:0:(lov_io.c:1061:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1788572632.705156:0:242485:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:0.0:1788572632.705159:0:242485:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572632.705162:0:242485:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572632.705164:0:242485:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572632.705165:0:242485:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572632.705168:0:242485:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788572632.705170:0:242485:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000020:00000001:0.0:1788572632.705173:0:242485:0:(cl_object.c:143:cl_object_top()) ffff8fc4accf8c38 -> ffff8fc4accf8f58 00000020:00000001:0.0:1788572632.705175:0:242485:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572632.705179:0:242485:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572632.705180:0:242485:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572632.705181:0:242485:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705189:0:242485:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705191:0:242485:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446620673502989264 : -123400206562352 : ffff8fc4a685cbd0) 00000020:00000001:0.0:1788572632.705193:0:242485:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1788572632.705195:0:242485:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705197:0:242485:0:(lov_io.c:1191:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705199:0:242485:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705201:0:242485:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1788572632.705203:0:242485:0:(lov_io.c:1334:lov_io_lock()) Process entered 00020000:00000001:0.0:1788572632.705205:0:242485:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572632.705206:0:242485:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1788572632.705209:0:242485:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1788572632.705211:0:242485:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705212:0:242485:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705215:0:242485:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705217:0:242485:0:(lov_io.c:1335:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705219:0:242485:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1788572632.705220:0:242485:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705223:0:242485:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705225:0:242485:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:0.0:1788572632.705228:0:242485:0:(lov_io.c:1340:lov_io_start()) Process entered 00020000:00000001:0.0:1788572632.705230:0:242485:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572632.705231:0:242485:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1788572632.705234:0:242485:0:(osc_io.c:1099:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1788572632.705238:0:242485:0:(osc_cache.c:3326:osc_cache_writeback_range()) Process entered 00000008:00000020:0.0:1788572632.705244:0:242485:0:(osc_cache.c:3466:osc_cache_writeback_range()) obj ffff8fc4a7c43c00 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:0.0:1788572632.705249:0:242485:0:(osc_cache.c:3481:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572632.705252:0:242485:0:(osc_io.c:1168:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705263:0:242485:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705265:0:242485:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705267:0:242485:0:(lov_io.c:1341:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705269:0:242485:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705271:0:242485:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1788572632.705277:0:242485:0:(lov_io.c:1882:lov_io_fsync_end()) Process entered 00020000:00000001:0.0:1788572632.705280:0:242485:0:(lov_io.c:1346:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1788572632.705281:0:242485:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1788572632.705283:0:242485:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1788572632.705284:0:242485:0:(lov_io.c:1356:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705286:0:242485:0:(lov_io.c:1893:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1788572632.705288:0:242485:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1788572632.705290:0:242485:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1788572632.705293:0:242485:0:(lov_io.c:1434:lov_io_unlock()) Process entered 00020000:00000001:0.0:1788572632.705302:0:242485:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572632.705303:0:242485:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1788572632.705305:0:242485:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1788572632.705306:0:242485:0:(lov_io.c:1368:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705308:0:242485:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705309:0:242485:0:(lov_io.c:1437:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1788572632.705310:0:242485:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1788572632.705312:0:242485:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:0.0:1788572632.705313:0:242485:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1788572632.705315:0:242485:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1788572632.705317:0:242485:0:(lov_io.c:1415:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1788572632.705318:0:242485:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572632.705318:0:242485:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1788572632.705319:0:242485:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1788572632.705320:0:242485:0:(lov_io.c:1362:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705321:0:242485:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705322:0:242485:0:(lov_io.c:1426:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1788572632.705323:0:242485:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1788572632.705324:0:242485:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705325:0:242485:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572632.705327:0:242485:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572632.705329:0:242485:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572632.705330:0:242485:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572632.705331:0:242485:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572632.705333:0:242485:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00020000:00000001:0.0:1788572632.705335:0:242485:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572632.705337:0:242485:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572632.705339:0:242485:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572632.705341:0:242485:0:(vvp_io.c:304:vvp_io_fini()) [0x2400013a6:0x10fc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572632.705343:0:242485:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572632.705344:0:242485:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572632.705346:0:242485:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000080:00000001:0.0:1788572632.705347:0:242485:0:(file.c:5332:cl_sync_file_range()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572632.705350:0:242485:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572632.705355:0:242485:0:(rw.c:1480:ll_writepages()) Process entered 00000080:00000001:0.0:1788572632.705356:0:242485:0:(file.c:5295:cl_sync_file_range()) Process entered 00000020:00000001:0.0:1788572632.705357:0:242485:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572632.705358:0:242485:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788572632.705360:0:242485:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000020:00000001:0.0:1788572632.705362:0:242485:0:(cl_object.c:143:cl_object_top()) ffff8fc4acb5af58 -> ffff8fc4acb5af58 00000020:00000001:0.0:1788572632.705363:0:242485:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572632.705364:0:242485:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572632.705369:0:242485:0:(vvp_io.c:1826:vvp_io_init()) [0x2400013a6:0x10fd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572632.705372:0:242485:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572632.705373:0:242485:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572632.705374:0:242485:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572632.705376:0:242485:0:(lov_object.c:1590:lov_io_init()) [0x2400013a6:0x10fd:0x0]io ffff8fc487d8f590 type 6 ignore/verify layout 1/0 00020000:00000002:0.0:1788572632.705378:0:242485:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff8fc4b17f38b8) owner 0000000000000000/ffff8fc4a0a28000 00020000:00000001:0.0:1788572632.705380:0:242485:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572632.705381:0:242485:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572632.705383:0:242485:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572632.705384:0:242485:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705385:0:242485:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572632.705387:0:242485:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572632.705389:0:242485:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705392:0:242485:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572632.705393:0:242485:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572632.705394:0:242485:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572632.705395:0:242485:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff8fc4b17f38b8) owner 0000000000000000/ffff8fc4a0a28000 00000020:00000001:0.0:1788572632.705396:0:242485:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705397:0:242485:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1788572632.705398:0:242485:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1788572632.705399:0:242485:0:(lov_io.c:1061:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1788572632.705400:0:242485:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:0.0:1788572632.705402:0:242485:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572632.705404:0:242485:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572632.705404:0:242485:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572632.705405:0:242485:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572632.705407:0:242485:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788572632.705408:0:242485:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000020:00000001:0.0:1788572632.705409:0:242485:0:(cl_object.c:143:cl_object_top()) ffff8fc4acb5ad00 -> ffff8fc4acb5af58 00000020:00000001:0.0:1788572632.705410:0:242485:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572632.705412:0:242485:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572632.705413:0:242485:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572632.705414:0:242485:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705415:0:242485:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705417:0:242485:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446620673502989264 : -123400206562352 : ffff8fc4a685cbd0) 00000020:00000001:0.0:1788572632.705418:0:242485:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1788572632.705419:0:242485:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705421:0:242485:0:(lov_io.c:1191:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705422:0:242485:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705426:0:242485:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1788572632.705427:0:242485:0:(lov_io.c:1334:lov_io_lock()) Process entered 00020000:00000001:0.0:1788572632.705428:0:242485:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572632.705428:0:242485:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1788572632.705429:0:242485:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1788572632.705430:0:242485:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705431:0:242485:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705432:0:242485:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705433:0:242485:0:(lov_io.c:1335:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705434:0:242485:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1788572632.705435:0:242485:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705436:0:242485:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705437:0:242485:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:0.0:1788572632.705437:0:242485:0:(lov_io.c:1340:lov_io_start()) Process entered 00020000:00000001:0.0:1788572632.705438:0:242485:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572632.705439:0:242485:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1788572632.705439:0:242485:0:(osc_io.c:1099:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1788572632.705440:0:242485:0:(osc_cache.c:3326:osc_cache_writeback_range()) Process entered 00000008:00000020:0.0:1788572632.705442:0:242485:0:(osc_cache.c:3466:osc_cache_writeback_range()) obj ffff8fc4b8731dc0 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:0.0:1788572632.705445:0:242485:0:(osc_cache.c:3481:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572632.705446:0:242485:0:(osc_io.c:1168:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705447:0:242485:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705448:0:242485:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705449:0:242485:0:(lov_io.c:1341:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705450:0:242485:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705451:0:242485:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1788572632.705451:0:242485:0:(lov_io.c:1882:lov_io_fsync_end()) Process entered 00020000:00000001:0.0:1788572632.705452:0:242485:0:(lov_io.c:1346:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1788572632.705453:0:242485:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1788572632.705453:0:242485:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1788572632.705454:0:242485:0:(lov_io.c:1356:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705455:0:242485:0:(lov_io.c:1893:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1788572632.705456:0:242485:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1788572632.705457:0:242485:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1788572632.705458:0:242485:0:(lov_io.c:1434:lov_io_unlock()) Process entered 00020000:00000001:0.0:1788572632.705458:0:242485:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572632.705459:0:242485:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1788572632.705460:0:242485:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1788572632.705461:0:242485:0:(lov_io.c:1368:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705462:0:242485:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705463:0:242485:0:(lov_io.c:1437:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1788572632.705463:0:242485:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1788572632.705467:0:242485:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:0.0:1788572632.705468:0:242485:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1788572632.705469:0:242485:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1788572632.705470:0:242485:0:(lov_io.c:1415:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1788572632.705471:0:242485:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572632.705472:0:242485:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1788572632.705473:0:242485:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1788572632.705473:0:242485:0:(lov_io.c:1362:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705474:0:242485:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705475:0:242485:0:(lov_io.c:1426:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1788572632.705476:0:242485:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1788572632.705477:0:242485:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705478:0:242485:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572632.705478:0:242485:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572632.705479:0:242485:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572632.705480:0:242485:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572632.705480:0:242485:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572632.705481:0:242485:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00020000:00000001:0.0:1788572632.705482:0:242485:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572632.705483:0:242485:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572632.705484:0:242485:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572632.705485:0:242485:0:(vvp_io.c:304:vvp_io_fini()) [0x2400013a6:0x10fd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572632.705486:0:242485:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572632.705487:0:242485:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572632.705490:0:242485:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000080:00000001:0.0:1788572632.705491:0:242485:0:(file.c:5332:cl_sync_file_range()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572632.705492:0:242485:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572632.705495:0:242485:0:(rw.c:1480:ll_writepages()) Process entered 00000080:00000001:0.0:1788572632.705495:0:242485:0:(file.c:5295:cl_sync_file_range()) Process entered 00000020:00000001:0.0:1788572632.705496:0:242485:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572632.705497:0:242485:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788572632.705498:0:242485:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000020:00000001:0.0:1788572632.705500:0:242485:0:(cl_object.c:143:cl_object_top()) ffff8fc4acb5adc8 -> ffff8fc4acb5adc8 00000020:00000001:0.0:1788572632.705501:0:242485:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788572632.705501:0:242485:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788572632.705502:0:242485:0:(vvp_io.c:1826:vvp_io_init()) [0x2400013a6:0x10fe:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788572632.705504:0:242485:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572632.705504:0:242485:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788572632.705505:0:242485:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788572632.705507:0:242485:0:(lov_object.c:1590:lov_io_init()) [0x2400013a6:0x10fe:0x0]io ffff8fc487d8f590 type 6 ignore/verify layout 1/0 00020000:00000002:0.0:1788572632.705510:0:242485:0:(lov_object.c:1256:lov_conf_freeze()) To take share lov(ffff8fc4a6fb07c0) owner 0000000000000000/ffff8fc4a0a28000 00020000:00000001:0.0:1788572632.705515:0:242485:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788572632.705516:0:242485:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788572632.705517:0:242485:0:(lov_io.c:476:lov_io_mirror_init()) Process entered 00020000:00000001:0.0:1788572632.705518:0:242485:0:(lov_io.c:487:lov_io_mirror_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705519:0:242485:0:(lov_io.c:837:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788572632.705521:0:242485:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788572632.705521:0:242485:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705522:0:242485:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572632.705523:0:242485:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788572632.705524:0:242485:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00020000:00000002:0.0:1788572632.705524:0:242485:0:(lov_object.c:1264:lov_conf_thaw()) To release share lov(ffff8fc4a6fb07c0) owner 0000000000000000/ffff8fc4a0a28000 00000020:00000001:0.0:1788572632.705526:0:242485:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705527:0:242485:0:(cl_io.c:823:cl_io_loop()) Process entered 00000020:00000001:0.0:1788572632.705527:0:242485:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00020000:00000001:0.0:1788572632.705528:0:242485:0:(lov_io.c:1061:lov_io_iter_init()) Process entered 00020000:00200000:0.0:1788572632.705529:0:242485:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00020000:00000002:0.0:1788572632.705530:0:242485:0:(lov_offset.c:253:lov_stripe_intersects()) [0->-1] -> [(0) 0->-1 (0)] 00020000:00000001:0.0:1788572632.705532:0:242485:0:(lov_io.c:158:lov_sub_get()) Process entered 00020000:00000001:0.0:1788572632.705533:0:242485:0:(lov_io.c:105:lov_io_sub_init()) Process entered 00000020:00000001:0.0:1788572632.705533:0:242485:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788572632.705535:0:242485:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788572632.705536:0:242485:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000020:00000001:0.0:1788572632.705537:0:242485:0:(cl_object.c:143:cl_object_top()) ffff8fc4acb5ab70 -> ffff8fc4acb5adc8 00000020:00000001:0.0:1788572632.705538:0:242485:0:(cl_io.c:125:__cl_io_init()) Process entered 00000020:00000001:0.0:1788572632.705541:0:242485:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788572632.705542:0:242485:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000020:00000001:0.0:1788572632.705543:0:242485:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705544:0:242485:0:(lov_io.c:149:lov_io_sub_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705545:0:242485:0:(lov_io.c:186:lov_sub_get()) Process leaving (rc=18446620673502989264 : -123400206562352 : ffff8fc4a685cbd0) 00000020:00000001:0.0:1788572632.705546:0:242485:0:(cl_io.c:399:cl_io_iter_init()) Process entered 00000020:00000001:0.0:1788572632.705547:0:242485:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705548:0:242485:0:(lov_io.c:1191:lov_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705549:0:242485:0:(cl_io.c:411:cl_io_iter_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705550:0:242485:0:(cl_io.c:314:cl_io_lock()) Process entered 00020000:00000001:0.0:1788572632.705550:0:242485:0:(lov_io.c:1334:lov_io_lock()) Process entered 00020000:00000001:0.0:1788572632.705551:0:242485:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572632.705553:0:242485:0:(cl_io.c:314:cl_io_lock()) Process entered 00000020:00000001:0.0:1788572632.705555:0:242485:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1788572632.705556:0:242485:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705558:0:242485:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705563:0:242485:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705565:0:242485:0:(lov_io.c:1335:lov_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705566:0:242485:0:(cl_io.c:286:cl_lockset_lock()) Process entered 00000020:00000001:0.0:1788572632.705567:0:242485:0:(cl_io.c:295:cl_lockset_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705568:0:242485:0:(cl_io.c:334:cl_io_lock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705571:0:242485:0:(cl_io.c:528:cl_io_start()) Process entered 00020000:00000001:0.0:1788572632.705571:0:242485:0:(lov_io.c:1340:lov_io_start()) Process entered 00020000:00000001:0.0:1788572632.705572:0:242485:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572632.705573:0:242485:0:(cl_io.c:528:cl_io_start()) Process entered 00000008:00000001:0.0:1788572632.705573:0:242485:0:(osc_io.c:1099:osc_io_fsync_start()) Process entered 00000008:00000001:0.0:1788572632.705574:0:242485:0:(osc_cache.c:3326:osc_cache_writeback_range()) Process entered 00000008:00000020:0.0:1788572632.705576:0:242485:0:(osc_cache.c:3466:osc_cache_writeback_range()) obj ffff8fc4b87016c0 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:0.0:1788572632.705579:0:242485:0:(osc_cache.c:3481:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788572632.705580:0:242485:0:(osc_io.c:1168:osc_io_fsync_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705583:0:242485:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705584:0:242485:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705585:0:242485:0:(lov_io.c:1341:lov_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705587:0:242485:0:(cl_io.c:540:cl_io_start()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705588:0:242485:0:(cl_io.c:555:cl_io_end()) Process entered 00020000:00000001:0.0:1788572632.705589:0:242485:0:(lov_io.c:1882:lov_io_fsync_end()) Process entered 00020000:00000001:0.0:1788572632.705590:0:242485:0:(lov_io.c:1346:lov_io_end_wrapper()) Process entered 00000020:00000001:0.0:1788572632.705591:0:242485:0:(cl_io.c:555:cl_io_end()) Process entered 00000020:00000001:0.0:1788572632.705592:0:242485:0:(cl_io.c:563:cl_io_end()) Process leaving 00020000:00000001:0.0:1788572632.705594:0:242485:0:(lov_io.c:1356:lov_io_end_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705596:0:242485:0:(lov_io.c:1893:lov_io_fsync_end()) Process leaving 00000020:00000001:0.0:1788572632.705596:0:242485:0:(cl_io.c:563:cl_io_end()) Process leaving 00000020:00000001:0.0:1788572632.705598:0:242485:0:(cl_io.c:352:cl_io_unlock()) Process entered 00020000:00000001:0.0:1788572632.705598:0:242485:0:(lov_io.c:1434:lov_io_unlock()) Process entered 00020000:00000001:0.0:1788572632.705599:0:242485:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572632.705600:0:242485:0:(cl_io.c:352:cl_io_unlock()) Process entered 00000020:00000001:0.0:1788572632.705601:0:242485:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00020000:00000001:0.0:1788572632.705602:0:242485:0:(lov_io.c:1368:lov_io_unlock_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705603:0:242485:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705605:0:242485:0:(lov_io.c:1437:lov_io_unlock()) Process leaving 00000020:00000001:0.0:1788572632.705607:0:242485:0:(cl_io.c:373:cl_io_unlock()) Process leaving 00000020:00000001:0.0:1788572632.705608:0:242485:0:(cl_io.c:449:cl_io_rw_advance()) Process entered 00000020:00000001:0.0:1788572632.705610:0:242485:0:(cl_io.c:465:cl_io_rw_advance()) Process leaving 00000020:00000001:0.0:1788572632.705612:0:242485:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00020000:00000001:0.0:1788572632.705613:0:242485:0:(lov_io.c:1415:lov_io_iter_fini()) Process entered 00020000:00000001:0.0:1788572632.705615:0:242485:0:(lov_io.c:1312:lov_io_call()) Process entered 00000020:00000001:0.0:1788572632.705616:0:242485:0:(cl_io.c:429:cl_io_iter_fini()) Process entered 00000020:00000001:0.0:1788572632.705617:0:242485:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00020000:00000001:0.0:1788572632.705624:0:242485:0:(lov_io.c:1362:lov_io_iter_fini_wrapper()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705626:0:242485:0:(lov_io.c:1329:lov_io_call()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1788572632.705628:0:242485:0:(lov_io.c:1426:lov_io_iter_fini()) Process leaving 00000020:00000001:0.0:1788572632.705629:0:242485:0:(cl_io.c:435:cl_io_iter_fini()) Process leaving 00000020:00000001:0.0:1788572632.705630:0:242485:0:(cl_io.c:873:cl_io_loop()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572632.705631:0:242485:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788572632.705632:0:242485:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788572632.705632:0:242485:0:(lov_io.c:74:lov_io_sub_fini()) Process entered 00000020:00000001:0.0:1788572632.705633:0:242485:0:(cl_io.c:71:cl_io_fini()) Process entered 00000020:00000001:0.0:1788572632.705634:0:242485:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572632.705634:0:242485:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00020000:00000001:0.0:1788572632.705636:0:242485:0:(lov_io.c:82:lov_io_sub_fini()) Process leaving 00020000:00000001:0.0:1788572632.705637:0:242485:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788572632.705639:0:242485:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788572632.705640:0:242485:0:(vvp_io.c:304:vvp_io_fini()) [0x2400013a6:0x10fe:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:0.0:1788572632.705642:0:242485:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788572632.705644:0:242485:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000020:00001000:0.0:1788572632.705645:0:242485:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000080:00000001:0.0:1788572632.705646:0:242485:0:(file.c:5332:cl_sync_file_range()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788572632.705648:0:242485:0:(rw.c:1554:ll_writepages()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572633.409191:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572633.409195:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572633.409198:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572634.433038:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572634.433058:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572634.433062:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572636.032907:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4307995886, at 4307995904 00000100:00000010:0.0:1788572636.032921:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab4e0. 00000100:00080000:0.0:1788572637.313045:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572637.313054:0:241211:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572637.313063:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572637.313068:0:241211:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572637.313074:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788572637.313080:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=12 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000100:00000001:0.0:1788572637.313085:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572637.313095:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572637.313120:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572637.313128:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8fc4b3b56800. 00000100:00000001:0.0:1788572637.313145:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572637.313165:0:241211:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID req@ffff8fc4b80c3b80 x1875440516474880/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572637.313230:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572637.313236:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572637.313248:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_00_00+0] req@ffff8fc4b80c3b80 x1875440516474880/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572637.313281:0:241211:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572637.313285:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572637.313292:0:241211:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572637.313299:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788572637.313304:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=16 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000100:00000001:0.0:1788572637.313309:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572637.313311:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572637.313315:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572637.313318:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8fc4b3b56100. 00000100:00000001:0.0:1788572637.313322:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572637.313334:0:241211:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID req@ffff8fc4b80c2a00 x1875440516475008/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572637.313383:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572637.313393:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572637.313415:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_00_01+1] req@ffff8fc4b80c2a00 x1875440516475008/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572637.313439:0:241211:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572637.313445:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572637.313452:0:241211:0:(pinger.c:247:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572637.313463:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572637.313481:0:241211:0:(pinger.c:247:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572637.313496:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13334) 00000100:00000001:0.0:1788572637.313509:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572637.313512:0:2400:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572637.313519:0:2400:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572637.313530:0:2400:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4b80c2a00 x1875440516475008/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572637.313576:0:2400:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572637.313581:0:2400:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572637.313589:0:2400:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572637.313595:0:2400:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572637.313607:0:2400:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572637.313623:0:2400:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4b80c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:51d6aedc-ea7c-426a-b495-22e82c13ee75:2400:1875440516475008:192.168.202.128@tcp:400:kworker.0 00000100:00000001:0.0:1788572637.313633:0:2400:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572637.313644:0:2400:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572637.313649:0:2400:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572637.313657:0:2400:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572637.313669:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8fc487b53000. 02000000:00000001:0.0:1788572637.313689:0:2400:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572637.313698:0:2400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572637.313704:0:2400:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915738. 00000400:00000010:0.0:1788572637.313711:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed95d8. 00000100:00000200:0.0:1788572637.313717:0:2400:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440516475008, portal 10 00000100:00000001:0.0:1788572637.313721:0:2400:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572637.313725:0:2400:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673797007872 : -123399912543744 : ffff8fc4b80c2a00) 00000100:00000040:0.0:1788572637.313743:0:2400:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4b80c2a00 x1875440516475008/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572723 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572637.313758:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572637.313765:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572637.313769:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788572637.313775:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440516475008, offset 0 00000400:00000200:0.0:1788572637.313787:0:2400:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572637.313835:0:2400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572637.313843:0:2400:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572637.313849:0:2400:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572637.313855:0:2400:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572637.313857:0:2400:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572637.313862:0:2400:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572637.313868:0:2400:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545369:545369:255:4294967295] 192.168.202.128@tcp LPNI seq info [545292:545292:7:4294967295] 00000400:00000200:0.0:1788572637.313877:0:2400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572637.313884:0:2400:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572637.313890:0:2400:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572637.313896:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56b00. 00000800:00000200:0.0:1788572637.313905:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (5) 00000800:00000200:0.0:1788572637.313917:0:2400:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.128@tcp ip 192.168.202.128:988 00000800:00000200:0.0:1788572637.313924:0:2400:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4b3b56b00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1788572637.313930:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572637.313936:0:2400:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788572637.313944:0:2400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572637.313947:0:2400:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4307997210 00000100:00000001:0.0:1788572637.313968:0:2400:0:(niobuf.c:685:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, simply extend its deadline from 4307997203 00000100:00000001:0.0:1788572637.313977:0:2400:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4307997210 00000100:00000001:0.0:1788572637.313986:0:2400:0:(niobuf.c:685:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, simply extend its deadline from 4307997203 00000100:00000001:0.0:1788572637.313990:0:2400:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572637.313995:0:2400:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572637.314000:0:2400:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572637.314003:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572637.314009:0:2400:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572637.314013:0:2400:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572637.314018:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572637.314021:0:2400:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572637.314023:0:2400:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572637.314025:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572637.339923:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4307997210, at 4307997211 00000100:00000010:0.0:1788572637.339939:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674900. 00000100:00000001:0.0:1788572638.336951:0:2400:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572638.336961:0:2400:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1788572638.336985:0:2400:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788572622/real 1788572622] req@ffff8fc4ac9f4000 x1875440516473600/t0(0) o400->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572638 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572638.337005:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572638.337008:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572638.337012:0:2400:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc488009160. 00000400:00000200:0.0:1788572638.337022:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4b385c770 00000400:00000010:0.0:1788572638.337026:0:2400:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4b385c770. 00000100:00000001:0.0:1788572638.337031:0:2400:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572638.337035:0:2400:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4000 x1875440516473600/t0(0) o400->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572638 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572638.337044:0:2400:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4000 x1875440516473600/t0(0) o400->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572638 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572638.337050:0:2400:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572638.337052:0:2400:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572638.337055:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572638.337056:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572638.337060:0:2400:0:(client.c:2555:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8fc4ac9f4000 x1875440516473600/t0(0) o400->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572638 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572638.337066:0:2400:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1788572638.337069:0:2400:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8fc4912d1800 already not connected (conn 10, was 8): CONNECTING 00000100:00000001:0.0:1788572638.337072:0:2400:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1788572638.337073:0:2400:0:(client.c:2563:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572638.337078:0:2400:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572638.337079:0:2400:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=59 : 59 : 3b) 00000100:00000001:0.0:1788572638.337081:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572638.337082:0:2400:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572638.337085:0:2400:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572638.337086:0:2400:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572638.337095:0:2400:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572638.337099:0:2400:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4000 x1875440516473600/t0(0) o400->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572638 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572638.337114:0:2400:0:(client.c:2198:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:0.0:1788572638.337117:0:2400:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572638.337121:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572638.337122:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572638.337125:0:2400:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4000 x1875440516473600/t0(0) o400->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572638 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572638.337131:0:2400:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:51d6aedc-ea7c-426a-b495-22e82c13ee75:2400:1875440516473600:192.168.202.128@tcp:400:kworker.0 00000100:00000001:0.0:1788572638.337136:0:2400:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572638.337138:0:2400:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572638.337141:0:2400:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4000 x1875440516473600/t0(0) o400->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1788572638 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572638.337146:0:2400:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572638.337150:0:2400:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572638.337154:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8fc487c84c00. 02000000:00000001:0.0:1788572638.337159:0:2400:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572638.337162:0:2400:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572638.337163:0:2400:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788572638.337166:0:2400:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572638.337168:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8fc39da39b00. 02000000:00000001:0.0:1788572638.337172:0:2400:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572638.337174:0:2400:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572638.337176:0:2400:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4000. 00000100:00000001:0.0:1788572638.337179:0:2400:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572638.337180:0:2400:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572638.337182:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572638.529041:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572638.529046:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572638.529050:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:0.0:1788572639.521047:0:265481:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572642.459957:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308002330, at 4308002331 00000100:00000010:0.0:1788572642.459975:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abae0. 00000020:00000001:0.0:1788572643.649100:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572643.649104:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572643.649107:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572645.504955:0:2398:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572645.504969:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572645.504973:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572645.504978:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572645.504983:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572645.504988:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572645.504990:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572646.528923:0:2398:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572646.528939:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572646.528942:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572646.528948:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572646.528952:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572646.528958:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572646.528962:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572647.553678:0:2386:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.202.128:988 00000020:00000001:0.0:1788572648.768996:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572648.769000:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572648.769004:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572651.392901:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308011246, at 4308011264 00000100:00000010:0.0:1788572651.392923:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea75a0. 00000800:00000200:0.0:1788572651.648938:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788572651.648947:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788572651.648952:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788572651.649224:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572651.649245:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572651.649251:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572651.649254:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572652.698909:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308012569, at 4308012570 00000100:00000010:0.0:1788572652.698926:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab840. 00000020:00000001:0.0:1788572653.889037:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572653.889048:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572653.889053:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788572656.272203:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572656.272209:0:241211:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572656.272213:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788572656.272216:0:241211:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1788572656.272224:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788572656.272229:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=16 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000100:00000001:0.0:1788572656.272234:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572656.272237:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572656.272241:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572656.272251:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8fc4b3b56c00. 00000100:00000001:0.0:1788572656.272256:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572656.272261:0:241211:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID req@ffff8fc4b80c2300 x1875440516476288/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572656.272269:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572656.272272:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572656.272277:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_00_00+0] req@ffff8fc4b80c2300 x1875440516476288/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572656.272297:0:241211:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572656.272299:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572656.272301:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572656.272302:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572656.272304:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572656.272307:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572656.272310:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 12894 00000400:00000200:0.0:1788572656.272315:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572656.272320:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00000040:0.0:1788572656.272324:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572656.272333:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572656.272338:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572656.272343:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572656.272349:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572656.272352:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572656.272357:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572656.272362:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572656.272369:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572656.272374:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572656.272394:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572656.272398:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572656.272404:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572656.272410:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572656.272415:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572656.272422:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572656.272429:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572656.272434:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572656.272441:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572656.272448:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572656.272453:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572656.272459:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572656.272467:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572656.272471:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572656.272478:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788572656.272484:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=20 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572656.272490:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572656.272493:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572656.272497:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572656.272498:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572656.272500:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572656.272505:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788572656.272510:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572656.272528:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516476416/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572656.272541:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572656.272544:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572656.272550:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516476416/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572656.272570:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572656.272575:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572656.272579:0:241211:0:(pinger.c:247:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788572656.272582:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572656.272587:0:241211:0:(pinger.c:247:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1788572656.272591:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13353) 00000100:00000001:0.0:1788572656.272609:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572656.272613:0:2400:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572656.272618:0:2400:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572656.272625:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572656.298915:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308016169, at 4308016170 00000100:00000010:0.0:1788572656.298930:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674480. 00000100:00000001:0.0:1788572657.280964:0:2398:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572657.280974:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572657.280975:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572657.280979:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572657.280981:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572657.280984:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572657.280986:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572658.304945:0:2398:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572658.304961:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572658.304963:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572658.304970:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572658.304973:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572658.304984:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572658.304992:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572659.009006:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572659.009009:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572659.009011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572659.328935:0:2398:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572659.328951:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572659.328955:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572659.328963:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572659.328967:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572659.328976:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572659.328982:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572660.352937:0:2398:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572660.352955:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572660.352960:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572660.352967:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572660.352970:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572660.352976:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572660.352980:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572661.377017:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572661.377024:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572661.377028:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572661.377035:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572661.377041:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572661.377047:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572661.377052:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572661.377056:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572661.377061:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572661.395937:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308021266, at 4308021267 00000100:00000010:0.0:1788572661.395956:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abba0. 00000100:00000001:0.0:1788572662.400883:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572662.400897:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572662.400900:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572662.400910:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572662.400916:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572662.400921:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572662.400925:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572662.400929:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572662.400934:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572663.424885:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572663.424901:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572663.424905:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572663.424911:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572663.424914:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572663.424919:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572663.424925:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572663.424927:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572663.424929:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572664.129002:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572664.129005:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572664.129007:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572664.448882:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572664.448886:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572664.448888:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572664.448891:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572664.448896:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572664.448899:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572664.448900:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572664.448902:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572664.448903:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572665.472875:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572665.472880:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572665.472883:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572665.472887:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572665.472891:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572665.472895:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572665.472900:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572665.472904:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572665.472906:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572666.496892:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572666.496900:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572666.496902:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572666.496908:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572666.496913:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572666.496917:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572666.496920:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572666.496924:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572666.496928:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572666.497279:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572666.497287:0:2400:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572666.497291:0:2400:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572666.497306:0:2400:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4700 x1875440516476672/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572666.497319:0:2400:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572666.497324:0:2400:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572666.497331:0:2400:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572666.497337:0:2400:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572666.497343:0:2400:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572666.497355:0:2400:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:51d6aedc-ea7c-426a-b495-22e82c13ee75:2400:1875440516476672:192.168.202.128@tcp:400:kworker.0 00000100:00000001:0.0:1788572666.497360:0:2400:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572666.497366:0:2400:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572666.497370:0:2400:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572666.497373:0:2400:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572666.497388:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8fc485fa9200. 02000000:00000001:0.0:1788572666.497391:0:2400:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572666.497398:0:2400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572666.497402:0:2400:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915420. 00000400:00000010:0.0:1788572666.497408:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9908. 00000100:00000200:0.0:1788572666.497414:0:2400:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875440516476672, portal 10 00000100:00000001:0.0:1788572666.497417:0:2400:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572666.497419:0:2400:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605322496 : -123400104229120 : ffff8fc4ac9f4700) 00000100:00000040:0.0:1788572666.497427:0:2400:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4700 x1875440516476672/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572752 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572666.497437:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572666.497440:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572666.497442:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9990. 00000100:00000200:0.0:1788572666.497445:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875440516476672, offset 0 00000400:00000200:0.0:1788572666.497451:0:2400:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572666.497458:0:2400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572666.497466:0:2400:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572666.497470:0:2400:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572666.497482:0:2400:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572666.497486:0:2400:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572666.497489:0:2400:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572666.497506:0:2400:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545382:545382:248:4294967295] 192.168.202.128@tcp LPNI seq info [545305:545305:-2:4294967295] 00000400:00000200:0.0:1788572666.497511:0:2400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572666.497519:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572666.497523:0:2400:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788572666.497527:0:2400:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572666.497530:0:2400:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308026386 00000100:00000010:0.0:1788572666.497535:0:2400:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788572666.497542:0:2400:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572666.497548:0:2400:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308026386 00000100:00000010:0.0:1788572666.497552:0:2400:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50f60. 00000100:00000001:0.0:1788572666.497555:0:2400:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572666.497565:0:2400:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572666.497568:0:2400:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572666.497572:0:2400:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572666.497575:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572666.497579:0:2400:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572666.497582:0:2400:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=31 : 31 : 1f) 00000100:00000001:0.0:1788572666.497586:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572666.497588:0:2400:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572666.497591:0:2400:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572666.497595:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572666.515895:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308026386, at 4308026387 00000100:00000010:0.0:1788572666.515905:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788572668.544980:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572668.544985:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572668.544988:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572668.544993:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572668.544998:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572668.545108:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572668.545112:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572668.545133:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572668.545136:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572669.249119:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572669.249127:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572669.249133:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572669.568939:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572669.568948:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572669.568950:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572669.568954:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572669.568957:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572669.568961:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572669.568964:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572669.568967:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572669.568982:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572670.592914:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572670.592920:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572670.592922:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572670.592927:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572670.592931:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572670.592934:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572670.592947:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572670.592950:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572670.592953:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572671.616924:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572671.616932:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572671.616937:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572671.616944:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572671.616949:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572671.616955:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572671.616961:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572671.616967:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572671.616973:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572671.635902:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308031506, at 4308031507 00000100:00000010:0.0:1788572671.635934:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abcc0. 00000100:00000001:0.0:1788572672.640933:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572672.640944:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572672.640947:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572672.640979:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572672.640983:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572672.640986:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572672.640989:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572672.640991:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572672.640994:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572673.664917:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572673.664924:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572673.664926:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572673.664932:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572673.664936:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572673.664940:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572673.664943:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572673.664946:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572673.664950:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572674.369061:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572674.369065:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572674.369068:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572676.737182:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572676.737188:0:2400:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572676.737194:0:2400:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572676.737199:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572676.755907:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308036626, at 4308036627 00000100:00000010:0.0:1788572676.755922:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916748a0. 00000020:00000001:0.0:1788572679.489034:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572679.489038:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572679.489041:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572681.857019:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572681.857030:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572681.857036:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572681.857044:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572681.857052:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572681.857058:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572681.857063:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572681.857065:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572681.857071:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572681.875898:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308041746, at 4308041747 00000100:00000010:0.0:1788572681.875926:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab480. 00000400:00000100:0.0:1788572682.369115:0:2386:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.128:988 00000400:00000100:0.0:1788572682.369182:0:2386:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788572682.369192:0:2386:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572682.369196:0:2386:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788572682.369200:0:2386:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572682.369205:0:2386:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788572682.369209:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 224 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572682.369213:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793000. 00000400:00000200:0.0:1788572682.369219:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572682.369223:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572682.369230:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc6e8 00000400:00000010:0.0:1788572682.369233:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc6e8. 00000100:00000001:0.0:1788572682.369238:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.369246:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac821c00 x1875440516475392/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.369268:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.369271:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516475392/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.369278:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.369280:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572682.369291:0:2386:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.369300:0:2386:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788572682.369318:0:2386:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788572682.369322:0:2386:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1788572682.369360:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.369363:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788572682.369368:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572682.369376:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572682.369381:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc220 00000400:00000010:0.0:1788572682.369389:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc220. 00000100:00000001:0.0:1788572682.369395:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.369403:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4b80c1c00 x1875440516476416/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572666 ref 2 fl UnregRPC:EXNU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.369425:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.369430:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c1c00 x1875440516476416/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572666 ref 2 fl UnregRPC:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.369442:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.369447:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572682.369456:0:2386:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.369460:0:2386:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788572682.369466:0:2386:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788572682.369473:0:2386:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1788572682.369504:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.369508:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788572682.369517:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572682.369532:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572682.369543:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed5d8 00000400:00000010:0.0:1788572682.369554:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed5d8. 00000100:00000001:0.0:1788572682.369565:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.369589:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5f80 x1875440516476544/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572747 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.369616:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.369626:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5f80 x1875440516476544/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572747 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.369647:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.369651:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572682.369664:0:2386:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.369670:0:2386:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788572682.369679:0:2386:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788572682.369686:0:2386:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1788572682.369727:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.369734:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788572682.369743:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572682.369755:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572682.369766:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9990 00000400:00000010:0.0:1788572682.369771:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9990. 00000100:00000001:0.0:1788572682.369780:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.369793:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4700 x1875440516476672/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572752 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.369855:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.369864:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4700 x1875440516476672/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572752 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.369898:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.369904:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572682.369920:0:2386:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.369929:0:2386:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788572682.369942:0:2386:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788572682.369950:0:2386:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1788572682.369982:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.369988:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788572682.369996:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572682.370007:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572682.370017:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76edd48 00000400:00000010:0.0:1788572682.370022:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76edd48. 00000100:00000001:0.0:1788572682.370030:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.370041:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7b80 x1875440516476800/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.370080:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.370092:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7b80 x1875440516476800/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572757 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.370155:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.370162:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572682.370179:0:2386:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.370189:0:2386:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788572682.370203:0:2386:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788572682.370212:0:2386:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1788572682.370246:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.370250:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788572682.370258:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572682.370269:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572682.370279:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4b385c2a8 00000400:00000010:0.0:1788572682.370285:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4b385c2a8. 00000100:00000001:0.0:1788572682.370292:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.370302:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516476928/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572762 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.370347:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.370358:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516476928/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572762 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.370383:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.370391:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572682.370406:0:2386:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.370416:0:2386:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788572682.370431:0:2386:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788572682.370439:0:2386:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1788572682.370478:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.370484:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788572682.370491:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572682.370501:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572682.370510:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed908 00000400:00000010:0.0:1788572682.370524:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed908. 00000100:00000001:0.0:1788572682.370533:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.370544:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516477056/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.370592:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.370602:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516477056/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572767 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.370637:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.370645:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000200:0.0:1788572682.370660:0:2386:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.370669:0:2386:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788572682.370682:0:2386:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788572682.370689:0:2386:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1788572682.370724:0:2386:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572682.370732:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788572682.370740:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788572682.370754:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788572682.370765:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed220 00000400:00000010:0.0:1788572682.370771:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed220. 00000100:00000001:0.0:1788572682.370780:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.370790:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7480 x1875440516477184/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572692 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.370851:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.370861:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7480 x1875440516477184/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572692 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.370897:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.370905:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572682.370924:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 224 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572682.370932:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc488504c00. 00000400:00000200:0.0:1788572682.370940:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572682.370953:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572682.370993:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76edb28 00000400:00000010:0.0:1788572682.371000:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76edb28. 00000100:00000001:0.0:1788572682.371008:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.371021:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac823800 x1875440516475520/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.371055:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.371065:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516475520/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.371087:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.371094:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572682.371111:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572682.371120:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc39da39300. 00000400:00000200:0.0:1788572682.371129:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572682.371145:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572682.371169:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4b385c110 00000400:00000010:0.0:1788572682.371177:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4b385c110. 00000100:00000001:0.0:1788572682.371186:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.371199:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a7100 x1875440516475648/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572667 ref 2 fl UnregRPC:EXNU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.371246:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.371257:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516475648/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572667 ref 2 fl UnregRPC:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.371298:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.371306:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572682.371322:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572682.371330:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc39da39600. 00000400:00000200:0.0:1788572682.371338:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572682.371351:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572682.371362:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4b385c220 00000400:00000010:0.0:1788572682.371368:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4b385c220. 00000100:00000001:0.0:1788572682.371376:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.371396:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a6a00 x1875440516475776/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572667 ref 2 fl UnregRPC:EXNU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.371436:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.371448:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6a00 x1875440516475776/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572667 ref 2 fl UnregRPC:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.371484:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.371491:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572682.371509:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572682.371518:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc39da39c00. 00000400:00000200:0.0:1788572682.371528:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572682.371542:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572682.371551:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4b385c6e8 00000400:00000010:0.0:1788572682.371558:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4b385c6e8. 00000100:00000001:0.0:1788572682.371566:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.371575:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a4a80 x1875440516475904/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572667 ref 2 fl UnregRPC:EXNU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.371616:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.371625:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516475904/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572667 ref 2 fl UnregRPC:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.371645:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.371650:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572682.371664:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 224 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572682.371672:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc488504500. 00000400:00000200:0.0:1788572682.371682:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572682.371694:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572682.371703:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76eddd0 00000400:00000010:0.0:1788572682.371709:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76eddd0. 00000100:00000001:0.0:1788572682.371716:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.371725:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a5c00 x1875440516476032/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.371748:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.371766:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516476032/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.371804:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.371837:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572682.371854:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 224 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572682.371864:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793a00. 00000400:00000200:0.0:1788572682.371873:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572682.371886:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572682.371894:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fca18 00000400:00000010:0.0:1788572682.371900:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fca18. 00000100:00000001:0.0:1788572682.371908:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.371917:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a7800 x1875440516476160/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.371965:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.371979:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516476160/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.372010:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.372017:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572682.372030:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 224 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572682.372038:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc39da39000. 00000400:00000200:0.0:1788572682.372044:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572682.372055:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572682.372061:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4b385cdd0 00000400:00000010:0.0:1788572682.372067:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4b385cdd0. 00000100:00000001:0.0:1788572682.372073:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572682.372084:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c2300 x1875440516476288/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572742 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.372131:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.372142:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c2300 x1875440516476288/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 0 dl 1788572742 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.372170:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.372186:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572682.372220:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572682.372226:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572682.372231:0:2399:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1788572682.372244:0:2399:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572656/real 1788572682] req@ffff8fc4b80c2300 x1875440516476288/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572682.372293:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572682.372299:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572682.372305:0:2399:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc488009898. 00000400:00000200:0.0:1788572682.372314:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4b385c660 00000400:00000010:0.0:1788572682.372320:0:2399:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4b385c660. 00000100:00000001:0.0:1788572682.372329:0:2399:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572682.372339:0:2399:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c2300 x1875440516476288/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572682.372369:0:2399:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c2300 x1875440516476288/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.372405:0:2399:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572682.372410:0:2399:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572682.372418:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572682.372424:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572682.372437:0:2399:0:(client.c:2555:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8fc4b80c2300 x1875440516476288/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.372475:0:2399:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1788572682.372481:0:2399:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8fc485ade000 already not connected (conn 4, was 3): DISCONN 00000100:00000001:0.0:1788572682.372493:0:2399:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1788572682.372498:0:2399:0:(client.c:2563:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572682.372513:0:2399:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c2300 x1875440516476288/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572742 ref 1 fl Rpc:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.372551:0:2399:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551611 : -5 : 0xfffffffffffffffb) 00000100:00000001:0.0:1788572682.372560:0:2399:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572682.372567:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572682.372594:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572682.372608:0:2399:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c2300 x1875440516476288/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572742 ref 1 fl Interpret:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572682.372642:0:2399:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:51d6aedc-ea7c-426a-b495-22e82c13ee75:2399:1875440516476288:192.168.202.128@tcp:400:kworker.0 00000100:00000001:0.0:1788572682.372659:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.372667:0:2399:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572682.372676:0:2399:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c2300 x1875440516476288/t0(0) o400->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572742 ref 1 fl Complete:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572682.372711:0:2399:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572682.372715:0:2399:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572682.372721:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8fc489d92c00. 02000000:00000001:0.0:1788572682.372732:0:2399:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572682.372737:0:2399:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572682.372741:0:2399:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=14 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788572682.372747:0:2399:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572682.372751:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8fc4b3b56c00. 02000000:00000001:0.0:1788572682.372759:0:2399:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572682.372764:0:2399:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572682.372769:0:2399:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c2300. 00000100:00000001:0.0:1788572682.372776:0:2399:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572682.372781:0:2399:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572682.372787:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.372792:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572682.372796:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572682.372803:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.372835:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572682.880889:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572682.880898:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572682.880900:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572682.880905:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572682.880909:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572682.880912:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572682.880915:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572682.880917:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572682.880942:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572683.904950:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572683.904959:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572683.904962:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572683.904968:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572683.904972:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572683.904976:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572683.904980:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572683.904983:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572683.904985:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572684.609105:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572684.609108:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572684.609112:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572684.928940:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572684.928952:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572684.928955:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572684.928962:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572684.928967:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572684.928972:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572684.928975:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572684.928979:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572684.928982:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572686.208886:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308046065, at 4308046080 00000100:00000010:0.0:1788572686.208904:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab7e0. 00000800:00000200:0.0:1788572689.537096:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788572689.728986:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572689.728990:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572689.728992:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572691.328880:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308051182, at 4308051200 00000100:00000010:0.0:1788572691.328897:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abde0. 00000100:00000001:0.0:1788572692.096943:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572692.096948:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572692.096952:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572692.096959:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 1 seconds 00000100:00000001:0.0:1788572692.096966:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572692.096970:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.609563:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572692.609571:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572692.609576:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572692.609588:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5180 x1875440516478464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572692.609612:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572692.609616:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572692.609624:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572692.609631:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572692.609638:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572692.609649:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516478464:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788572692.609660:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572692.609669:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572692.609675:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572692.609682:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572692.609692:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788572692.609696:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572692.609707:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572692.609715:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915790. 00000400:00000010:0.0:1788572692.609727:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9990. 00000100:00000200:0.0:1788572692.609741:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516478464, portal 25 00000100:00000001:0.0:1788572692.609748:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572692.609753:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605325184 : -123400104226432 : ffff8fc4ac9f5180) 00000100:00000040:0.0:1788572692.609766:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5180 x1875440516478464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572692.609783:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572692.609790:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572692.609796:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9880. 00000100:00000200:0.0:1788572692.609804:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516478464, offset 0 00000400:00000200:0.0:1788572692.609846:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572692.609857:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572692.609866:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572692.609879:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572692.609898:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572692.609906:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572692.609915:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572692.609927:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545396:545396:256:4294967295] 192.168.202.128@tcp LPNI seq info [545319:545319:8:4294967295] 00000400:00000200:0.0:1788572692.609939:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572692.609957:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572692.609967:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572692.609983:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56c00. 00000800:00000200:0.0:1788572692.609998:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788572692.610040:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572692.610048:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572692.610052:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4308056302 00000100:00000010:0.0:1788572692.610060:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788572692.610070:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572692.610080:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4308056302 00000100:00000010:0.0:1788572692.610084:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50180. 00000100:00000001:0.0:1788572692.610088:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572692.610092:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572692.610096:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.610100:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.610103:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572692.610112:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572692.610124:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572692.610127:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572692.610132:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572692.610134:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572692.610139:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5c00 x1875440516478592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572692.610160:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572692.610163:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572692.610167:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572692.610170:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572692.610179:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572692.610185:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516478592:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788572692.610195:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572692.610198:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572692.610201:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572692.610204:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572692.610208:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788572692.610210:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572692.610216:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572692.610220:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915318. 00000400:00000010:0.0:1788572692.610225:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9198. 00000100:00000200:0.0:1788572692.610230:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516478592, portal 10 00000100:00000001:0.0:1788572692.610232:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572692.610235:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605327872 : -123400104223744 : ffff8fc4ac9f5c00) 00000100:00000040:0.0:1788572692.610245:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5c00 x1875440516478592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572692.610253:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572692.610257:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572692.610260:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788572692.610263:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516478592, offset 0 00000400:00000200:0.0:1788572692.610271:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572692.610373:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572692.610383:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572692.610390:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572692.610398:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572692.610406:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572692.610414:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572692.610423:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545397:545397:255:4294967295] 192.168.202.128@tcp LPNI seq info [545320:545320:7:4294967295] 00000400:00000200:0.0:1788572692.610434:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572692.610452:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572692.610465:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572692.610480:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788572692.610492:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788572692.610511:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572692.610519:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788572692.610527:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572692.610531:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308052499 00000100:00000001:0.0:1788572692.610537:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308056302 00000100:00000001:0.0:1788572692.610542:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308052499 00000100:00000001:0.0:1788572692.610548:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308056302 00000100:00000001:0.0:1788572692.610554:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572692.610560:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.610565:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.610570:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572692.610576:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572692.610580:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572692.610586:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572692.610589:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572692.610592:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.610596:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572692.610600:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572692.610601:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572692.610603:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572692.610605:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572692.610608:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572692.610617:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5880 x1875440516478720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572692.610634:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572692.610638:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572692.610642:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572692.610647:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572692.610652:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572692.610661:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516478720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788572692.610669:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572692.610686:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572692.610691:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572692.610696:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572692.610703:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d400. 02000000:00000001:0.0:1788572692.610709:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572692.610721:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572692.610728:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915f78. 00000400:00000010:0.0:1788572692.610736:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9c38. 00000100:00000200:0.0:1788572692.610744:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516478720, portal 10 00000100:00000001:0.0:1788572692.610748:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572692.610752:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605326976 : -123400104224640 : ffff8fc4ac9f5880) 00000100:00000040:0.0:1788572692.610762:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5880 x1875440516478720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572692.610781:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572692.610787:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572692.610794:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9550. 00000100:00000200:0.0:1788572692.610798:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516478720, offset 0 00000400:00000200:0.0:1788572692.610807:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572692.610841:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572692.610848:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572692.610853:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572692.610859:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572692.610865:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572692.610872:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572692.610880:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545398:545398:254:4294967295] 192.168.202.128@tcp LPNI seq info [545321:545321:6:4294967295] 00000400:00000200:0.0:1788572692.610891:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572692.610908:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572692.610920:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572692.610929:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56000. 00000800:00000200:0.0:1788572692.610941:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788572692.610956:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572692.610964:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788572692.610979:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572692.610985:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308052507 00000100:00000001:0.0:1788572692.610992:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308056302 00000100:00000001:0.0:1788572692.610996:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308052507 00000100:00000001:0.0:1788572692.610998:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308056302 00000100:00000001:0.0:1788572692.611000:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572692.611004:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.611010:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.611019:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572692.611027:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572692.611032:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572692.611039:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572692.611043:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572692.611047:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572692.611056:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7b80 x1875440516478848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572692.611080:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572692.611083:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572692.611089:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572692.611094:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572692.611100:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572692.611109:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516478848:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788572692.611117:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572692.611124:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572692.611127:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572692.611133:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572692.611140:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788572692.611146:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572692.611168:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572692.611176:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915d10. 00000400:00000010:0.0:1788572692.611185:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9b28. 00000100:00000200:0.0:1788572692.611192:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516478848, portal 4 00000100:00000001:0.0:1788572692.611204:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572692.611207:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335936 : -123400104215680 : ffff8fc4ac9f7b80) 00000100:00000040:0.0:1788572692.611217:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7b80 x1875440516478848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572692.611225:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572692.611229:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572692.611232:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9bb0. 00000100:00000200:0.0:1788572692.611235:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516478848, offset 0 00000400:00000200:0.0:1788572692.611240:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572692.611244:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572692.611248:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572692.611251:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572692.611254:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572692.611257:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572692.611259:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572692.611263:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545399:545399:253:4294967295] 192.168.202.128@tcp LPNI seq info [545322:545322:5:4294967295] 00000400:00000200:0.0:1788572692.611268:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572692.611275:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572692.611280:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572692.611284:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788572692.611288:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788572692.611295:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572692.611298:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788572692.611302:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572692.611305:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308052647 00000100:00000001:0.0:1788572692.611307:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308056302 00000100:00000001:0.0:1788572692.611310:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308052647 00000100:00000001:0.0:1788572692.611312:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308056302 00000100:00000001:0.0:1788572692.611314:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572692.611336:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.611339:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.611341:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572692.611345:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572692.611347:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572692.611349:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572692.611351:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572692.611352:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572692.611357:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6d80 x1875440516478976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572692.611365:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572692.611367:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572692.611369:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572692.611371:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572692.611374:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572692.611377:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516478976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788572692.611381:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572692.611383:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572692.611385:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572692.611387:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572692.611391:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788572692.611393:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572692.611397:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572692.611400:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915ec8. 00000400:00000010:0.0:1788572692.611406:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9908. 00000100:00000200:0.0:1788572692.611409:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516478976, portal 4 00000100:00000001:0.0:1788572692.611411:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572692.611413:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605332352 : -123400104219264 : ffff8fc4ac9f6d80) 00000100:00000040:0.0:1788572692.611419:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6d80 x1875440516478976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572692.611427:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572692.611429:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572692.611432:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed92a8. 00000100:00000200:0.0:1788572692.611434:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516478976, offset 0 00000400:00000200:0.0:1788572692.611445:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572692.611449:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572692.611453:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572692.611456:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572692.611459:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572692.611461:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572692.611464:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572692.611467:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545400:545400:252:4294967295] 192.168.202.128@tcp LPNI seq info [545323:545323:4:4294967295] 00000400:00000200:0.0:1788572692.611471:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572692.611478:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572692.611483:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572692.611487:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788572692.611491:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788572692.611498:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572692.611501:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788572692.611516:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572692.611522:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308052649 00000100:00000001:0.0:1788572692.611525:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308056302 00000100:00000001:0.0:1788572692.611527:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308052649 00000100:00000001:0.0:1788572692.611530:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308056302 00000100:00000001:0.0:1788572692.611532:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572692.611535:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.611538:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.611540:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572692.611544:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572692.611545:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572692.611547:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572692.611549:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572692.611551:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572692.611553:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572693.120943:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572693.120978:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572693.120986:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572693.120994:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 1 seconds 00000100:00000001:0.0:1788572693.121002:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572693.121006:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572693.633382:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572693.633392:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572693.633402:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572693.633423:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572692/real 1788572693] req@ffff8fc4ac9f5180 x1875440516478464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572693.633487:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572693.633493:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572693.633500:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915790. 00000400:00000200:0.0:1788572693.633524:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9990 00000400:00000010:0.0:1788572693.633528:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9990. 00000100:00000001:0.0:1788572693.633535:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572693.633546:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516478464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572693.633570:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516478464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.633597:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572693.633606:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.633614:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572693.633618:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572693.633634:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516478464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.633662:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572693.633685:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516478464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.633724:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572693.633742:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.633752:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572693.633783:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.633790:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572693.633797:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572693.633844:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572693.633855:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788572693.633860:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572693.633873:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516478464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572693.633898:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516478464:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788572693.633919:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572693.633927:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572693.633937:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516478464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.633949:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572693.633955:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572693.633966:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788572693.633986:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572693.633993:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572693.633996:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788572693.634000:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572693.634004:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788572693.634010:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572693.634013:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572693.634015:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788572693.634020:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572693.634021:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634024:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572693.634027:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572693.634029:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572693.634031:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572693.634037:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572692/real 1788572693] req@ffff8fc4ac9f6d80 x1875440516478976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572693.634065:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572693.634067:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572693.634069:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915ec8. 00000400:00000200:0.0:1788572693.634073:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9908 00000400:00000010:0.0:1788572693.634075:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9908. 00000100:00000001:0.0:1788572693.634078:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572693.634083:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6d80 x1875440516478976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572693.634092:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6d80 x1875440516478976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634100:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572693.634101:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634104:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572693.634105:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572693.634109:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6d80 x1875440516478976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634117:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572693.634121:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6d80 x1875440516478976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634129:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572693.634132:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634134:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572693.634135:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634137:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572693.634139:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572693.634143:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572693.634147:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788572693.634150:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572693.634163:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6d80 x1875440516478976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572693.634172:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516478976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788572693.634178:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572693.634183:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572692/real 1788572693] req@ffff8fc4ac9f7b80 x1875440516478848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572693.634193:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572693.634195:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572693.634197:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915d10. 00000400:00000200:0.0:1788572693.634201:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9b28 00000400:00000010:0.0:1788572693.634202:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9b28. 00000100:00000001:0.0:1788572693.634205:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572693.634209:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7b80 x1875440516478848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572693.634217:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7b80 x1875440516478848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634223:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572693.634225:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634227:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572693.634228:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572693.634231:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7b80 x1875440516478848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634238:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572693.634242:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7b80 x1875440516478848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634249:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572693.634251:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634259:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572693.634261:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634263:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572693.634265:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572693.634268:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572693.634272:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788572693.634275:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572693.634279:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7b80 x1875440516478848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572693.634287:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516478848:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788572693.634292:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572693.634309:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572692/real 1788572693] req@ffff8fc4ac9f5880 x1875440516478720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572693.634319:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572693.634321:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572693.634324:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915f78. 00000400:00000200:0.0:1788572693.634327:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9c38 00000400:00000010:0.0:1788572693.634330:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9c38. 00000100:00000001:0.0:1788572693.634334:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572693.634337:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5880 x1875440516478720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572693.634346:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5880 x1875440516478720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634353:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572693.634355:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634357:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572693.634359:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572693.634363:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5880 x1875440516478720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634383:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572693.634389:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5880 x1875440516478720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634396:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572693.634398:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634400:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572693.634402:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634404:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572693.634406:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572693.634409:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572693.634414:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788572693.634417:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572693.634422:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5880 x1875440516478720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572693.634430:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516478720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788572693.634435:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788572693.634440:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788572692/real 1788572693] req@ffff8fc4ac9f5c00 x1875440516478592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572693.634450:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572693.634452:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572693.634454:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915318. 00000400:00000200:0.0:1788572693.634457:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9198 00000400:00000010:0.0:1788572693.634459:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9198. 00000100:00000001:0.0:1788572693.634462:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572693.634466:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5c00 x1875440516478592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572693.634488:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5c00 x1875440516478592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634496:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572693.634497:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634499:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572693.634501:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572693.634505:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5c00 x1875440516478592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634511:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572693.634515:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5c00 x1875440516478592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572712 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634521:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788572693.634524:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634526:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572693.634527:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634528:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788572693.634531:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788572693.634534:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788572693.634538:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788572693.634541:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788572693.634545:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5c00 x1875440516478592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572712 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572693.634552:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516478592:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788572693.634558:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634561:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572693.634564:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6d80 x1875440516478976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634584:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572693.634586:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572693.634588:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788572693.634594:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572693.634597:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572693.634598:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=4 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788572693.634601:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572693.634603:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37c00. 02000000:00000001:0.0:1788572693.634607:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572693.634608:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572693.634610:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6d80. 00000100:00000001:0.0:1788572693.634613:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572693.634614:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634616:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572693.634620:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7b80 x1875440516478848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634628:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572693.634629:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572693.634631:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788572693.634634:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572693.634635:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572693.634636:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=3 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788572693.634639:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572693.634640:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788572693.634643:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572693.634644:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572693.634646:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7b80. 00000100:00000001:0.0:1788572693.634648:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572693.634649:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634651:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572693.634654:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5880 x1875440516478720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572717 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634662:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572693.634663:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572693.634673:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d400. 02000000:00000001:0.0:1788572693.634677:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572693.634678:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572693.634680:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=16 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788572693.634682:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572693.634684:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34c00. 02000000:00000001:0.0:1788572693.634687:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572693.634689:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572693.634690:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5880. 00000100:00000001:0.0:1788572693.634693:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572693.634695:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634697:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572693.634702:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5c00 x1875440516478592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788572712 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572693.634709:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572693.634711:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572693.634712:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788572693.634718:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572693.634719:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572693.634721:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=12 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788572693.634723:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572693.634725:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34400. 02000000:00000001:0.0:1788572693.634728:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572693.634729:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572693.634730:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5c00. 00000100:00000001:0.0:1788572693.634732:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572693.634734:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634736:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572693.634747:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572693.634749:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572693.634752:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572693.634753:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572694.144967:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572694.144976:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572694.144979:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572694.144986:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 1 seconds 00000100:00000001:0.0:1788572694.144995:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572694.145027:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572694.849058:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572694.849064:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572694.849067:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572695.169043:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572695.169053:0:265052:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1788572695.169074:0:265052:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788572609/real 1788572609] req@ffff8fc4b80c2d80 x1875440516469504/t0(0) o101->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 912/66584 e 0 to 1 dl 1788572695 ref 2 fl Rpc:XPQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000400:00000001:0.0:1788572695.169171:0:265052:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572695.169177:0:265052:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572695.169184:0:265052:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915dc0. 00000400:00000200:0.0:1788572695.169197:0:265052:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4acd9e000 00000400:00000010:0.0:1788572695.169210:0:265052:0:(lib-lnet.h:286:lnet_md_free()) kfreed 'md': 632 at ffff8fc4acd9e000. 00000100:00000001:0.0:1788572695.169227:0:265052:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572695.169246:0:265052:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c2d80 x1875440516469504/t0(0) o101->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 912/66584 e 0 to 1 dl 1788572695 ref 2 fl Rpc:XPQr/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1788572695.169322:0:265052:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c2d80 x1875440516469504/t0(0) o101->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 912/66584 e 0 to 1 dl 1788572695 ref 2 fl Rpc:XPQU/600/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1788572695.169395:0:265052:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572695.169402:0:265052:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572695.169418:0:265052:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572695.169432:0:265052:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572695.169445:0:265052:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1788572695.169455:0:265052:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8fc486471000 already not connected (conn 13, was 9): DISCONN 00000100:00000001:0.0:1788572695.169466:0:265052:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1788572695.169471:0:265052:0:(client.c:2563:ptlrpc_expire_one_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572695.169482:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572695.169490:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572695.169507:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572695.169512:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572695.169523:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572695.169526:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572695.169528:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572695.169531:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572696.448882:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308056302, at 4308056320 00000100:00000010:0.0:1788572696.448902:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788572697.216987:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572697.217002:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572697.217017:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572697.217038:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572697.217051:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572697.217062:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.217079:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572697.217088:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.217098:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.217135:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572697.217147:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572697.217151:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572697.217156:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572697.217162:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572697.217167:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.217172:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572697.217176:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.217179:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.729321:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572697.729326:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572697.729329:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572697.729338:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516479104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.729350:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572697.729352:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572697.729354:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572697.729356:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572697.729358:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572697.729363:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516479104:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788572697.729366:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572697.729370:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572697.729373:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572697.729376:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572697.729388:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788572697.729392:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572697.729403:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572697.729409:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915dc0. 00000400:00000010:0.0:1788572697.729416:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9198. 00000100:00000200:0.0:1788572697.729425:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516479104, portal 25 00000100:00000001:0.0:1788572697.729428:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572697.729432:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788572697.729438:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516479104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.729446:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572697.729450:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572697.729452:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9c38. 00000100:00000200:0.0:1788572697.729454:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516479104, offset 0 00000400:00000200:0.0:1788572697.729464:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572697.729474:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572697.729484:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572697.729491:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572697.729497:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572697.729504:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572697.729508:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572697.729512:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545401:545401:256:4294967295] 192.168.202.128@tcp LPNI seq info [545324:545324:8:4294967295] 00000400:00000200:0.0:1788572697.729518:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572697.729529:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572697.729536:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572697.729542:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788572697.729561:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788572697.729582:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572697.729588:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572697.729592:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4308061421 00000100:00000010:0.0:1788572697.729597:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788572697.729604:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788572697.729616:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4308061421 00000100:00000010:0.0:1788572697.729623:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50960. 00000100:00000001:0.0:1788572697.729627:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788572697.729632:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572697.729637:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.729641:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.729644:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.729649:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572697.729652:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572697.729656:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572697.729660:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572697.729661:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572697.729666:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516479232/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.729675:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572697.729678:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572697.729684:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572697.729687:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572697.729691:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572697.729696:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516479232:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788572697.729701:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572697.729705:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572697.729708:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572697.729711:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572697.729714:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788572697.729718:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572697.729725:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572697.729729:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915318. 00000400:00000010:0.0:1788572697.729732:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9b28. 00000100:00000200:0.0:1788572697.729735:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516479232, portal 10 00000100:00000001:0.0:1788572697.729736:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572697.729737:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788572697.729746:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516479232/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.729771:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572697.729776:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572697.729780:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9908. 00000100:00000200:0.0:1788572697.729784:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516479232, offset 0 00000400:00000200:0.0:1788572697.729792:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572697.729799:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572697.729832:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572697.729837:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572697.729843:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572697.729850:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572697.729857:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572697.729866:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545402:545402:255:4294967295] 192.168.202.128@tcp LPNI seq info [545325:545325:7:4294967295] 00000400:00000200:0.0:1788572697.729874:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572697.729885:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572697.729893:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572697.729899:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56f00. 00000800:00000200:0.0:1788572697.729906:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788572697.729919:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572697.729927:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788572697.729935:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572697.729941:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308057619 00000100:00000001:0.0:1788572697.729947:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308061421 00000100:00000001:0.0:1788572697.729953:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308057619 00000100:00000001:0.0:1788572697.729958:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308061421 00000100:00000001:0.0:1788572697.729962:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572697.729964:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.729970:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.729973:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.729982:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572697.729994:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572697.729999:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572697.730001:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572697.730005:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.730010:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.730081:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572697.730085:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572697.730087:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572697.730096:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516479360/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.730123:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572697.730128:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572697.730133:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572697.730137:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572697.730144:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572697.730153:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516479360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788572697.730161:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572697.730165:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572697.730169:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572697.730173:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572697.730180:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788572697.730187:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572697.730207:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572697.730213:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915f78. 00000400:00000010:0.0:1788572697.730221:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9990. 00000100:00000200:0.0:1788572697.730227:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516479360, portal 10 00000100:00000001:0.0:1788572697.730237:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572697.730242:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788572697.730255:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516479360/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.730265:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572697.730271:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572697.730277:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9330. 00000100:00000200:0.0:1788572697.730282:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516479360, offset 0 00000400:00000200:0.0:1788572697.730299:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572697.730307:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572697.730315:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572697.730324:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572697.730331:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572697.730338:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572697.730345:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572697.730354:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545403:545403:254:4294967295] 192.168.202.128@tcp LPNI seq info [545326:545326:6:4294967295] 00000400:00000200:0.0:1788572697.730365:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572697.730374:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572697.730379:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572697.730384:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788572697.730395:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788572697.730414:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572697.730421:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788572697.730432:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572697.730437:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308057626 00000100:00000001:0.0:1788572697.730443:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308061421 00000100:00000001:0.0:1788572697.730450:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308057626 00000100:00000001:0.0:1788572697.730457:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308061421 00000100:00000001:0.0:1788572697.730464:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572697.730470:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.730477:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.730482:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.730492:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572697.730496:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572697.730502:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572697.730506:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572697.730509:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572697.730517:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7b80 x1875440516479488/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.730552:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572697.730556:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572697.730561:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572697.730566:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572697.730573:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572697.730581:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516479488:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788572697.730587:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572697.730593:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572697.730595:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572697.730599:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572697.730606:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d400. 02000000:00000001:0.0:1788572697.730611:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572697.730619:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572697.730623:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915d10. 00000400:00000010:0.0:1788572697.730628:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed96e8. 00000100:00000200:0.0:1788572697.730631:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516479488, portal 4 00000100:00000001:0.0:1788572697.730633:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572697.730635:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126857600 : -123400582694016 : ffff8fc4901a7b80) 00000100:00000040:0.0:1788572697.730641:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7b80 x1875440516479488/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.730648:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572697.730651:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572697.730653:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788572697.730655:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516479488, offset 0 00000400:00000200:0.0:1788572697.730660:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572697.730664:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572697.730668:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572697.730671:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572697.730674:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572697.730676:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572697.730679:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572697.730689:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545404:545404:253:4294967295] 192.168.202.128@tcp LPNI seq info [545327:545327:5:4294967295] 00000400:00000200:0.0:1788572697.730701:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572697.730717:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572697.730733:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572697.730739:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56200. 00000800:00000200:0.0:1788572697.730748:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788572697.730757:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572697.730761:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788572697.730767:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572697.730769:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308057766 00000100:00000001:0.0:1788572697.730773:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308061421 00000100:00000001:0.0:1788572697.730776:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308057766 00000100:00000001:0.0:1788572697.730779:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308061421 00000100:00000001:0.0:1788572697.730782:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572697.730784:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.730790:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.730792:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.730797:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572697.730800:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572697.730803:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572697.730805:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572697.730845:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788572697.730851:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5f80 x1875440516479616/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.730860:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572697.730861:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572697.730864:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788572697.730866:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572697.730868:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572697.730871:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516479616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788572697.730885:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788572697.730888:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788572697.730889:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572697.730890:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788572697.730894:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788572697.730896:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788572697.730900:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788572697.730903:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915ec8. 00000400:00000010:0.0:1788572697.730907:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788572697.730910:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516479616, portal 4 00000100:00000001:0.0:1788572697.730912:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788572697.730913:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126850432 : -123400582701184 : ffff8fc4901a5f80) 00000100:00000040:0.0:1788572697.730920:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5f80 x1875440516479616/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572697.730926:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788572697.730928:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788572697.730931:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788572697.730933:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516479616, offset 0 00000400:00000200:0.0:1788572697.730938:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788572697.730942:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572697.730946:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788572697.730950:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788572697.730953:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788572697.730955:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788572697.730957:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788572697.730960:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [545405:545405:252:4294967295] 192.168.202.128@tcp LPNI seq info [545328:545328:4:4294967295] 00000400:00000200:0.0:1788572697.730963:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572697.730970:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788572697.730974:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788572697.730978:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788572697.730982:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788572697.730995:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572697.730998:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788572697.731001:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788572697.731003:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308057769 00000100:00000001:0.0:1788572697.731006:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308061421 00000100:00000001:0.0:1788572697.731008:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308057769 00000100:00000001:0.0:1788572697.731010:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308061421 00000100:00000001:0.0:1788572697.731012:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788572697.731014:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.731017:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.731019:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572697.731023:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572697.731024:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788572697.731026:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572697.731028:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572697.731029:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572697.731031:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572698.240981:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572698.240995:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572698.240999:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572698.241009:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572698.241017:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572698.241023:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572698.241028:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.264909:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572699.264922:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572699.264928:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572699.264935:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572699.264941:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572699.264945:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572699.264950:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572699.969022:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572699.969026:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572699.969028:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572700.288931:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572700.288945:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572700.288967:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572700.288976:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572700.288980:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572700.288986:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572700.288991:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572700.289197:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572700.289199:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572700.289200:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572700.289202:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572700.289203:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572700.289205:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572700.289206:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572700.289207:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572700.289208:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788572700.801190:0:2387:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.128:988 00000400:00000100:0.0:1788572700.801228:0:2387:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788572700.801239:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572700.801243:0:2387:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788572700.801248:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572700.801262:0:2387:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788572700.801268:0:2387:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572700.801272:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788572700.801278:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572700.801283:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572700.801290:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9c38 00000400:00000010:0.0:1788572700.801294:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9c38. 00000100:00000001:0.0:1788572700.801300:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572700.801308:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a7800 x1875440516479104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572700.801335:0:2387:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572700.801339:0:2387:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516479104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572700.801345:0:2387:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572700.801347:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572700.801374:0:2387:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572700.801376:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56f00. 00000400:00000200:0.0:1788572700.801380:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572700.801383:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572700.801387:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9908 00000400:00000010:0.0:1788572700.801389:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9908. 00000100:00000001:0.0:1788572700.801392:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572700.801397:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a5c00 x1875440516479232/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572700.801407:0:2387:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572700.801427:0:2387:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516479232/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572700.801436:0:2387:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572700.801437:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572700.801443:0:2387:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572700.801446:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788572700.801451:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572700.801456:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572700.801460:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9330 00000400:00000010:0.0:1788572700.801463:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9330. 00000100:00000001:0.0:1788572700.801467:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572700.801476:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a5180 x1875440516479360/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572700.801487:0:2387:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572700.801491:0:2387:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516479360/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572700.801498:0:2387:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572700.801500:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572700.801510:0:2387:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572700.801515:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56200. 00000400:00000200:0.0:1788572700.801524:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572700.801548:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572700.801560:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788572700.801567:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788572700.801576:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572700.801588:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a7b80 x1875440516479488/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572700.801606:0:2387:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572700.801617:0:2387:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7b80 x1875440516479488/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572700.801635:0:2387:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572700.801640:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572700.801659:0:2387:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572700.801673:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788572700.801682:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572700.801703:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572700.801715:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788572700.801722:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788572700.801730:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572700.801741:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a5f80 x1875440516479616/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572700.801789:0:2387:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572700.801797:0:2387:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5f80 x1875440516479616/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572722 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572700.801884:0:2387:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572700.801891:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572701.313035:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572701.313051:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572701.313062:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572701.313078:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572701.313097:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572701.313118:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572701.313128:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572701.313147:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572701.313153:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572701.568895:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308061421, at 4308061440 00000100:00000010:0.0:1788572701.568913:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788572702.336951:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572702.336962:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572702.336966:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572702.336973:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572702.336980:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572702.336992:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572702.337004:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572702.337015:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572702.337030:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.337100:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572702.337110:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572702.337118:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572702.337127:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572702.337140:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572702.337151:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572702.337162:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572702.337171:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572702.337179:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572702.848990:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572702.849001:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572702.849005:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572702.849011:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572702.849020:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572702.849031:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13390 00000400:00000200:0.0:1788572702.849050:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572702.849065:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572702.849088:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572702.849101:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849114:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572702.849121:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849145:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572702.849157:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572702.849163:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572702.849172:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572702.849178:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572702.849183:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572702.849186:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572702.849203:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849212:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572702.849218:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849225:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572702.849233:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572702.849244:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849260:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572702.849270:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572702.849274:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572702.849282:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.849290:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572702.849311:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572702.849329:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788572702.849338:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572702.849344:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572702.849350:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.849359:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572702.849367:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572702.849378:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572702.849393:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572702.849405:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572702.849427:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516479744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572702.849463:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572702.849472:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572702.849498:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516479744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572702.849542:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572702.849549:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572702.849559:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572702.849570:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572702.849577:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572702.849584:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572702.849594:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13390 00000400:00000200:0.0:1788572702.849604:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572702.849613:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572702.849621:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572702.849634:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849642:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572702.849650:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849656:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572702.849664:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572702.849666:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572702.849668:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572702.849672:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572702.849674:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572702.849676:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572702.849681:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849683:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572702.849687:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849689:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572702.849692:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572702.849699:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849703:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572702.849714:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572702.849725:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572702.849728:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.849730:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572702.849736:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572702.849739:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.849742:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572702.849746:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788572702.849749:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=11 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572702.849752:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572702.849754:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.849756:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572702.849757:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572702.849760:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572702.849764:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788572702.849769:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572702.849776:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516479872/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572702.849785:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572702.849787:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572702.849791:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516479872/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572702.849800:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572702.849804:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572702.849835:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572702.849838:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572702.849840:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572702.849847:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572702.849854:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13390 00000400:00000200:0.0:1788572702.849863:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572702.849871:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572702.849877:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572702.849891:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849896:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572702.849902:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849907:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572702.849912:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572702.849916:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572702.849919:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572702.849923:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572702.849926:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572702.849931:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572702.849939:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849942:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572702.849945:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849952:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572702.849958:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572702.849963:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572702.849968:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572702.849972:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572702.849978:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572702.849982:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.849986:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572702.849990:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572702.849994:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.850000:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572702.850005:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788572702.850011:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=15 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572702.850016:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572702.850020:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.850026:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572702.850028:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572702.850033:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572702.850040:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788572702.850053:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572702.850065:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516480000/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572702.850096:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572702.850099:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572702.850109:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516480000/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572702.850126:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572702.850132:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572702.850139:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572702.850144:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572702.850147:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572702.850155:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572702.850161:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13390 00000400:00000200:0.0:1788572702.851171:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572702.851179:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572702.851184:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572702.851191:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.851196:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572702.851200:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.851204:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572702.851210:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572702.851213:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572702.851217:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572702.851220:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572702.851226:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572702.851234:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572702.851241:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.851246:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572702.851262:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.851267:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572702.851270:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572702.851275:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572702.851281:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572702.851286:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572702.851293:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572702.851301:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.851305:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572702.851309:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572702.851315:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.851317:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572702.851323:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788572702.851329:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572702.851337:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572702.851339:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.851341:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572702.851344:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572702.851348:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572702.851357:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788572702.851365:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572702.851375:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516480128/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572702.851419:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572702.851426:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572702.851435:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516480128/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572702.851458:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572702.851466:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572702.851478:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572702.851483:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572702.851493:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572702.851500:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572702.851507:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13390 00000400:00000200:0.0:1788572702.851517:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572702.851524:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572702.851528:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572702.851532:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.851536:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572702.851541:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.851545:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572702.851550:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572702.851553:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572702.851555:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572702.851559:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572702.851561:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572702.851565:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572702.851571:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.851573:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572702.851577:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572702.851580:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572702.851582:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572702.851586:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572702.851588:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572702.851589:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572702.851592:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572702.851593:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.851595:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572702.851596:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572702.851599:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.851600:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572702.851610:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788572702.851614:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572702.851617:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572702.851620:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572702.851623:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572702.851626:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572702.851629:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572702.851634:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788572702.851639:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572702.851645:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516480256/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572702.851669:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572702.851675:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572702.851681:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516480256/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572702.851690:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572702.851698:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13400) 00000100:00000001:0.0:1788572703.361020:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572703.361024:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572703.361026:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572703.361029:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572703.361032:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572703.361037:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572703.361042:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572703.361045:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572703.361051:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572703.361079:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572703.361084:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572703.361087:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572703.361091:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572703.361094:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572703.361096:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572703.361106:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572703.361108:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572703.361110:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572704.384959:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572704.384968:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572704.384973:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572704.384980:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572704.384993:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572704.385001:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572704.385024:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572704.385039:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572704.385045:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572704.385116:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572704.385130:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572704.385134:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572704.385137:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572704.385139:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572704.385141:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572704.385143:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572705.088955:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572705.088958:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572705.088961:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572705.408927:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572705.408938:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572705.408942:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572705.408946:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572705.408951:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572705.408956:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572705.408963:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572705.408989:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572705.408993:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572705.408996:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572705.409000:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572705.409020:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572705.409025:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572705.409032:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572705.409036:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572705.409040:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572706.432914:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572706.432921:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572706.432926:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572706.432935:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572706.432958:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572706.432965:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572706.432974:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572706.432991:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572706.432997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572706.433043:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572706.433048:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572706.433050:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572706.433053:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572706.433056:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572706.433059:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572706.433061:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572706.433063:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572706.433066:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572706.688905:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308066543, at 4308066560 00000100:00000010:0.0:1788572706.688914:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9aba20. 00000100:00000001:0.0:1788572707.456924:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572707.456932:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572707.456936:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572707.456943:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572707.456949:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572707.456965:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572707.456977:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572707.456982:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572707.456988:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.457007:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572707.457012:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572707.457016:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572707.457018:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572707.457021:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572707.457027:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572707.457033:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572707.457038:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572707.457041:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572707.969000:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572707.969025:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572707.969063:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572707.969069:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572707.969077:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572707.969091:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13395 00000400:00000200:0.0:1788572707.969115:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572707.969130:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572707.969138:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572707.969149:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.969160:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572707.969167:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.969175:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572707.969189:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572707.969194:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572707.969202:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572707.969208:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572707.969215:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572707.969230:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572707.969245:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.969248:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572707.969256:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.969264:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572707.969274:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572707.969277:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572707.969279:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572707.969282:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572707.969286:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572707.969289:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.969296:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572707.969299:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572707.969308:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788572707.969312:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572707.969323:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572707.969328:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.969332:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572707.969337:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572707.969343:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572707.969355:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788572707.969364:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572707.969381:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516480384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572707.969400:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572707.969409:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572707.969416:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516480384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572707.969453:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572707.969459:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572707.969473:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572707.969477:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572707.969480:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572707.969486:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572707.969490:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13395 00000400:00000200:0.0:1788572707.969504:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572707.969511:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572707.969515:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572707.969524:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.969530:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572707.969543:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.969552:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572707.969561:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572707.969569:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572707.969580:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572707.969587:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572707.969609:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572707.969620:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572707.969629:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.969634:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572707.969639:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.969645:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572707.969651:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572707.969658:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572707.969663:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572707.969668:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572707.969681:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572707.969687:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.969690:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572707.969696:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572707.969702:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.969704:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572707.969710:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788572707.970087:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=10 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572707.970092:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572707.970096:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.970101:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572707.970104:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572707.970110:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572707.970119:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572707.970126:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572707.970138:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516480512/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572707.970155:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572707.970161:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572707.970170:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516480512/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572707.970217:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572707.970223:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572707.970231:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572707.970234:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572707.970239:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572707.970246:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572707.970253:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13395 00000400:00000200:0.0:1788572707.970264:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572707.970272:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572707.970278:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572707.970286:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970293:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572707.970300:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970306:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572707.970313:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572707.970316:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572707.970319:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572707.970322:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572707.970325:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572707.970330:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572707.970337:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970342:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572707.970346:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970348:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572707.970351:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572707.970358:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970363:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572707.970376:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572707.970383:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572707.970389:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.970395:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572707.970400:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572707.970404:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.970409:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572707.970416:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3100. 00000020:00000040:0.0:1788572707.970424:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=14 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572707.970429:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572707.970433:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.970436:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572707.970440:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572707.970445:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572707.970454:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788572707.970463:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572707.970471:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3100 x1875440516480640/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572707.970488:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572707.970495:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572707.970506:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053a4b590] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3100 x1875440516480640/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572707.970526:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572707.970538:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572707.970545:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572707.970550:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572707.970555:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572707.970559:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572707.970567:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13395 00000400:00000200:0.0:1788572707.970577:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572707.970584:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572707.970589:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572707.970604:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970609:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572707.970615:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970621:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572707.970627:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572707.970630:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572707.970634:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572707.970638:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572707.970643:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572707.970649:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572707.970655:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970658:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572707.970665:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970671:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572707.970678:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572707.970682:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970687:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572707.970691:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572707.970696:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572707.970700:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.970705:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572707.970710:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572707.970715:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.970719:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572707.970725:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788572707.970730:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572707.970735:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572707.970739:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.970743:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572707.970746:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572707.970749:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572707.970757:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788572707.970770:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572707.970780:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516480768/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572707.970794:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572707.970799:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572707.970805:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516480768/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572707.970848:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572707.970854:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572707.970862:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572707.970868:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572707.970873:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572707.970880:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572707.970885:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13395 00000400:00000200:0.0:1788572707.970896:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572707.970901:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572707.970904:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572707.970907:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970910:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572707.970912:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970914:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572707.970917:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572707.970919:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572707.970921:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572707.970923:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572707.970926:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572707.970928:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572707.970932:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970934:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572707.970953:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970957:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572707.970960:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572707.970964:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572707.970966:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572707.970969:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572707.970972:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572707.970974:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.970976:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572707.970979:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572707.970982:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.970984:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572707.970988:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788572707.970991:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572707.970994:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572707.970996:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572707.970997:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572707.970999:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572707.971001:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572707.971007:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572707.971011:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572707.971017:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516480896/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572707.971026:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572707.971028:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572707.971032:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516480896/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572707.971041:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572707.971047:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13405) 00000100:00000001:0.0:1788572708.480971:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572708.480978:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572708.480981:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572708.481002:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572708.481013:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572708.481017:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.481031:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572708.481037:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.481043:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572708.481085:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572708.481089:0:2399:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1788572708.481106:0:2399:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788572622/real 1788572622] req@ffff8fc4ac9f5500 x1875440516473344/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572708 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788572708.481124:0:2399:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788572708.481126:0:2399:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788572708.481130:0:2399:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788572708.481136:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788572708.481139:0:2399:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788572708.481145:0:2399:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788572708.481150:0:2399:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5500 x1875440516473344/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572708 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788572708.481161:0:2399:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5500 x1875440516473344/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572708 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.481170:0:2399:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788572708.481172:0:2399:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.481175:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572708.481177:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788572708.481183:0:2399:0:(client.c:2555:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8fc4ac9f5500 x1875440516473344/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572708 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.481188:0:2399:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1788572708.481190:0:2399:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8fc486471000 already not connected (conn 16, was 9): DISCONN 00000100:00000001:0.0:1788572708.481193:0:2399:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1788572708.481194:0:2399:0:(client.c:2563:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.481200:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572708.481201:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=10 : 10 : a) 00000100:00000001:0.0:1788572708.481203:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572708.481211:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572708.481212:0:2399:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.481214:0:2399:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572708.481215:0:2399:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572708.481218:0:2399:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5500 x1875440516473344/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572708 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.481224:0:2399:0:(client.c:2198:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:0.0:1788572708.481226:0:2399:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.481227:0:2399:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788572708.481228:0:2399:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788572708.481231:0:2399:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5500 x1875440516473344/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572708 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788572708.481236:0:2399:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:51d6aedc-ea7c-426a-b495-22e82c13ee75:2399:1875440516473344:192.168.202.128@tcp:400:kworker.0 00000100:00000001:0.0:1788572708.481239:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572708.481241:0:2399:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572708.481243:0:2399:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5500 x1875440516473344/t0(0) o400->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1788572708 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572708.481247:0:2399:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788572708.481250:0:2399:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788572708.481253:0:2399:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8fc487997000. 02000000:00000001:0.0:1788572708.481257:0:2399:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788572708.481259:0:2399:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788572708.481260:0:2399:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=12 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788572708.481262:0:2399:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788572708.481264:0:2399:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8fc39da39e00. 02000000:00000001:0.0:1788572708.481267:0:2399:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788572708.481269:0:2399:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788572708.481270:0:2399:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5500. 00000100:00000001:0.0:1788572708.481274:0:2399:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788572708.481275:0:2399:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572708.481277:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572709.504962:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572709.504992:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572709.504999:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572709.505009:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572709.505015:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572709.505018:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572709.505042:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572709.505046:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572709.505050:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572710.209017:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572710.209020:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572710.209022:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572710.528968:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572710.528974:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572710.528977:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572710.528981:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572710.528984:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572710.528986:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.528989:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572710.528991:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.528993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572710.529022:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572710.529026:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572710.529030:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572710.529033:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572710.529036:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572710.529040:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.529047:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572710.529051:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572710.529054:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572711.552927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572711.552936:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572711.552941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572711.552949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572711.552956:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572711.552963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572711.552970:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572711.552973:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572711.552977:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572711.808919:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308071661, at 4308071680 00000100:00000010:0.0:1788572711.808930:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674600. 00000100:00000001:0.0:1788572712.576917:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572712.576924:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572712.576927:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572712.576935:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572712.576942:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572712.576948:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572712.576953:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572712.576956:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572712.576961:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572712.576996:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572712.577004:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572712.577009:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572712.577014:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572712.577021:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572712.577025:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572712.577029:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572712.577030:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572712.577032:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572713.088908:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572713.088915:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572713.088918:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572713.088923:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572713.088929:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572713.088940:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13400 00000400:00000200:0.0:1788572713.088952:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572713.088963:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572713.088969:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572713.088976:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.088983:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572713.088987:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.088989:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572713.088993:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572713.089010:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572713.089013:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572713.089015:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572713.089017:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572713.089019:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572713.089022:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089024:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572713.089025:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089026:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572713.089029:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572713.089031:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089033:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572713.089035:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572713.089037:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572713.089039:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.089041:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572713.089043:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572713.089048:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788572713.089051:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572713.089054:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572713.089055:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.089057:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572713.089059:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572713.089063:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572713.089069:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788572713.089075:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572713.089082:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516481024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572713.089093:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572713.089097:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572713.089102:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516481024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572713.089136:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572713.089139:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572713.089141:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572713.089143:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572713.089144:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572713.089146:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572713.089148:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13400 00000400:00000200:0.0:1788572713.089153:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572713.089158:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572713.089161:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572713.089165:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089170:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572713.089174:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089178:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572713.089182:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572713.089186:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572713.089190:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572713.089193:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572713.089199:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572713.089204:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572713.089211:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089215:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572713.089218:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089221:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572713.089224:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572713.089228:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089230:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572713.089239:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572713.089245:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572713.089249:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.089252:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572713.089257:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572713.089260:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.089264:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572713.089268:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788572713.089271:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=9 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572713.089273:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572713.089274:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.089275:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572713.089277:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572713.089280:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572713.089286:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572713.089291:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572713.089297:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516481152/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572713.089308:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572713.089310:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572713.089314:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516481152/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572713.089322:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572713.089327:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572713.089332:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572713.089335:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572713.089338:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572713.089340:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572713.089345:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13400 00000400:00000200:0.0:1788572713.089352:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572713.089357:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572713.089361:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572713.089373:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089384:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572713.089389:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089394:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572713.089398:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572713.089399:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572713.089402:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572713.089406:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572713.089409:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572713.089415:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572713.089421:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089426:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572713.089432:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089435:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572713.089438:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572713.089444:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089448:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572713.089450:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572713.089456:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572713.089460:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.089463:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572713.089467:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572713.089471:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.089476:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572713.089483:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3100. 00000020:00000040:0.0:1788572713.089486:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=13 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572713.089490:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572713.089493:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.089496:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572713.089499:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572713.089502:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572713.089507:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572713.089516:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572713.089523:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3100 x1875440516481280/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572713.089534:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572713.089538:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572713.089543:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053a4b590] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3100 x1875440516481280/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572713.089550:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572713.089552:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572713.089554:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572713.089556:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572713.089557:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572713.089559:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572713.089565:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13400 00000400:00000200:0.0:1788572713.089574:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572713.089581:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572713.089586:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572713.089592:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089598:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572713.089603:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089608:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572713.089615:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572713.089618:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572713.089623:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572713.089629:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572713.089636:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572713.089641:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572713.089648:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089653:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572713.089667:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089673:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572713.089679:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572713.089685:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089689:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572713.089693:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572713.089699:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572713.089706:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.089713:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572713.089717:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572713.089724:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.089728:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572713.089735:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788572713.089740:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572713.089746:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572713.089749:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.089754:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572713.089758:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572713.089763:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572713.089770:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572713.089778:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572713.089789:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516481408/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572713.089861:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572713.089866:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572713.089877:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516481408/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572713.089898:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572713.089905:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572713.089912:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572713.089916:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572713.089926:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572713.089933:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572713.089939:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13400 00000400:00000200:0.0:1788572713.089966:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572713.089974:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572713.089980:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572713.089989:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.089994:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572713.089999:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.090003:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572713.090010:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572713.090014:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572713.090018:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572713.090021:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572713.090025:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572713.090029:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572713.090038:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.090044:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572713.090052:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572713.090057:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572713.090063:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572713.090071:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572713.090076:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572713.090081:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572713.090087:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572713.090093:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.090097:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572713.090101:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572713.090108:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.090113:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572713.090125:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788572713.090132:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572713.090138:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572713.090142:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572713.090147:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572713.090149:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572713.090152:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572713.090157:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788572713.090164:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572713.090175:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516481536/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572713.090187:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572713.090192:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572713.090202:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516481536/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572713.090219:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572713.090228:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13410) 00000100:00000001:0.0:1788572713.600900:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572713.600911:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572713.600917:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572713.600925:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572713.600934:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572713.600941:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572713.600958:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572713.600961:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572713.600964:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572714.624907:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572714.624924:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572714.624927:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572714.624936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572714.624940:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572714.624944:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572714.624948:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572714.624951:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572714.624955:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572714.625004:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572714.625009:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572714.625013:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572714.625017:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572714.625019:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572714.625024:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572714.625026:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572714.625027:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572714.625031:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572715.328977:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572715.328980:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572715.328983:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572715.648921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572715.648929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572715.648934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572715.648942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572715.648950:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572715.648955:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572715.649299:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572715.649301:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572715.649304:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788572716.161260:0:2386:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.128:988 00000400:00000100:0.0:1788572716.161318:0:2386:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788572716.161331:0:2386:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572716.161336:0:2386:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788572716.161342:0:2386:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572716.161347:0:2386:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788572716.161354:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572716.161359:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc39da39f00. 00000400:00000200:0.0:1788572716.161366:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572716.161371:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572716.161381:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4b385c440 00000400:00000010:0.0:1788572716.161384:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4b385c440. 00000100:00000001:0.0:1788572716.161391:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572716.161402:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c3480 x1875440516481024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572716.161455:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572716.161461:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c3480 x1875440516481024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572716.161469:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572716.161472:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572716.161488:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572716.161491:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc39da39a00. 00000400:00000200:0.0:1788572716.161495:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572716.161499:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572716.161503:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4b385c000 00000400:00000010:0.0:1788572716.161505:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4b385c000. 00000100:00000001:0.0:1788572716.161508:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572716.161513:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c3800 x1875440516481152/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572716.161522:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572716.161526:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c3800 x1875440516481152/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572716.161533:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572716.161535:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572716.161541:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572716.161544:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc39da39b00. 00000400:00000200:0.0:1788572716.161547:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572716.161552:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572716.161555:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4b385c990 00000400:00000010:0.0:1788572716.161558:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4b385c990. 00000100:00000001:0.0:1788572716.161561:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572716.161568:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c3100 x1875440516481280/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572716.161580:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572716.161586:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c3100 x1875440516481280/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572716.161605:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572716.161610:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572716.161621:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572716.161625:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc39da39200. 00000400:00000200:0.0:1788572716.161629:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572716.161636:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572716.161652:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4b385c4c8 00000400:00000010:0.0:1788572716.161657:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4b385c4c8. 00000100:00000001:0.0:1788572716.161662:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572716.161672:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c1880 x1875440516481408/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572716.161685:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572716.161692:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c1880 x1875440516481408/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572716.161705:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572716.161708:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572716.161717:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572716.161722:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc39da39500. 00000400:00000200:0.0:1788572716.161728:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572716.161752:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572716.161758:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4b385cb28 00000400:00000010:0.0:1788572716.161762:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4b385cb28. 00000100:00000001:0.0:1788572716.161768:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572716.161777:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c0000 x1875440516481536/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572738 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572716.161792:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572716.161799:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c0000 x1875440516481536/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572738 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572716.161857:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572716.161862:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572716.672903:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572716.672910:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572716.672913:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572716.672920:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572716.672925:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572716.672934:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572716.672944:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572716.672951:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572716.672967:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572716.928882:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308076781, at 4308076800 00000100:00000010:0.0:1788572716.928900:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674120. 00000100:00000001:0.0:1788572717.696904:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572717.696912:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572717.696916:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572717.696921:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572717.696927:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572717.696933:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572717.696947:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572717.696950:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572717.696954:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572717.696969:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572717.696974:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572717.696977:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572717.696979:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572717.696982:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572717.696985:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572717.696989:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572717.696990:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572717.696993:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572718.721004:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572718.721021:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572718.721025:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572718.721030:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572718.721035:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572718.721039:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572718.721046:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572718.721076:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572718.721081:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572718.721108:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572718.721114:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572718.721131:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572718.721136:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572718.721140:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572718.721142:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572718.721145:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572719.744978:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572719.744988:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572719.744991:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572719.744999:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572719.745001:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572719.745004:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572719.745010:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572720.448982:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572720.448986:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572720.448998:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572720.768945:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572720.768959:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572720.768963:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572720.768970:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572720.768975:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572720.768979:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572720.768989:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572720.769048:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572720.769053:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572720.769057:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572720.769062:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572720.769071:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572720.769077:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572720.769099:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572720.769102:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572720.769106:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572721.792948:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572721.792960:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572721.792964:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572721.792973:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572721.792979:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572721.792987:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572721.793016:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572721.793023:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572721.793032:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572721.793061:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572721.793080:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572721.793085:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572721.793090:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572721.793095:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572721.793101:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572721.793112:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572722.048894:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308081902, at 4308081920 00000100:00000010:0.0:1788572722.048916:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea77e0. 00000100:00000001:0.0:1788572722.816895:0:2399:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572722.816905:0:2399:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572722.816907:0:2399:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572722.816911:0:2399:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788572722.816915:0:2399:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572722.816920:0:2399:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572722.816928:0:2399:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572722.816958:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572722.816963:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572722.816967:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572722.816974:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572722.816980:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572722.816984:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572722.816987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572722.816990:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572722.816992:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572723.840910:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572723.840917:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572723.840919:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572723.840924:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572723.840929:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572723.840935:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572723.840942:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572723.840945:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572723.840950:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572723.840980:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572723.840991:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572723.840995:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572723.840999:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572723.841004:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572723.841009:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572723.841012:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572723.841016:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572723.841020:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572724.864925:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572724.864929:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572724.864934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572724.864941:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572724.864947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572724.864951:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572724.864954:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572724.864959:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572724.864965:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572724.864983:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572724.864986:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572724.864988:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572724.864990:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572724.864992:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572724.864997:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572724.865002:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572724.865006:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572724.865013:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572725.569030:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572725.569033:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572725.569036:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572725.888959:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572725.888967:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572725.888974:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572725.888989:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572725.889001:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572725.889009:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572725.889019:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572725.889024:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572725.889029:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572726.912865:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572726.912872:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572726.912874:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572726.912879:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572726.912885:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572726.912888:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572726.912891:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572726.912895:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572726.912899:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572726.912939:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572726.912943:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572726.912945:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572726.912949:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572726.912953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572726.912957:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572726.912960:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572726.912963:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572726.912969:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572727.169724:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308087022, at 4308087040 00000100:00000010:0.0:1788572727.169744:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea79c0. 00000100:00000001:0.0:1788572727.936926:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572727.936932:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572727.936934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572727.936942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572727.936947:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572727.936952:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572727.936957:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572727.936963:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572727.936969:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572727.936992:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572727.936997:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572727.937003:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572727.937010:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572727.937016:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572727.937022:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572727.937033:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572727.937035:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572727.937047:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572728.448887:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788572728.448901:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788572728.448908:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000100:00080000:0.0:1788572728.448982:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572728.448989:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572728.448992:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572728.448995:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572728.448999:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572728.449004:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13415 00000400:00000200:0.0:1788572728.449023:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572728.449031:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572728.449034:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572728.449036:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449040:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572728.449043:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449046:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572728.449049:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572728.449052:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572728.449055:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572728.449060:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572728.449063:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572728.449066:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572728.449073:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449077:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572728.449086:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449098:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572728.449105:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572728.449114:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449121:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572728.449140:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572728.449143:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572728.449154:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.449160:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572728.449163:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572728.449169:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788572728.449174:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572728.449181:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572728.449183:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.449190:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572728.449193:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572728.449198:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572728.449206:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788572728.449214:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572728.449229:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516482944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572728.449273:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572728.449276:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572728.449286:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516482944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572728.449335:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572728.449348:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572728.449359:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572728.449362:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572728.449368:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572728.449379:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572728.449390:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13415 00000400:00000200:0.0:1788572728.449400:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572728.449408:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572728.449416:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572728.449420:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449423:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572728.449452:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449457:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572728.449463:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572728.449467:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572728.449470:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572728.449475:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572728.449479:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572728.449485:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572728.449492:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449496:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572728.449501:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449508:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572728.449514:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572728.449517:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449520:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572728.449524:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572728.449533:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572728.449539:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.449553:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572728.449565:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572728.449573:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.449583:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572728.449591:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788572728.449600:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572728.449606:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572728.449611:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.449615:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572728.449620:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572728.449625:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572728.449637:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788572728.449648:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572728.449667:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516483072/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572728.449710:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572728.449717:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572728.449731:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516483072/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572728.449757:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572728.449764:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572728.449775:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572728.449779:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572728.449784:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572728.449793:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572728.449803:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13415 00000400:00000200:0.0:1788572728.449841:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572728.449847:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572728.449854:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572728.449862:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449868:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572728.449874:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449881:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572728.449886:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572728.449891:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572728.449899:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572728.449905:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572728.449913:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572728.449922:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572728.449937:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449948:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572728.449958:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.449963:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572728.449997:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572728.450008:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572728.450017:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572728.450022:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572728.450028:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572728.450034:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.450040:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572728.450045:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572728.450051:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.450056:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572728.450063:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788572728.450069:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572728.450073:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572728.450076:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.450081:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572728.450085:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572728.450088:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572728.450095:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572728.450101:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572728.450112:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516483200/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572728.450131:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572728.450136:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572728.450148:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516483200/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572728.450165:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572728.450171:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572728.450177:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572728.450182:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572728.450184:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572728.450188:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572728.450201:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13415 00000400:00000200:0.0:1788572728.450211:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572728.450218:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572728.450222:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572728.450230:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.450235:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572728.450239:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.450243:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572728.450248:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572728.450251:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572728.450255:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572728.450259:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572728.450262:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572728.450265:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572728.450273:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.450279:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572728.450284:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.450288:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572728.450291:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572728.450297:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572728.450301:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572728.450305:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572728.450311:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572728.450317:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.450321:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572728.450326:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572728.450332:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.450336:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572728.450343:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788572728.450348:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572728.450350:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572728.450360:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.450365:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572728.450368:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572728.450373:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572728.450380:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788572728.450387:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572728.450399:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516483328/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572728.450422:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572728.450439:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572728.450449:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516483328/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572728.450469:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572728.450475:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572728.450483:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572728.450487:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572728.450492:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572728.450497:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572728.450502:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13415 00000400:00000200:0.0:1788572728.450509:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572728.450517:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572728.450521:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572728.450529:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.450533:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572728.450538:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.450542:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572728.450548:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572728.450552:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572728.450555:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572728.450559:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572728.450564:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572728.450576:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572728.450585:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.450590:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572728.450596:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572728.450600:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572728.450607:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572728.450612:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572728.450618:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572728.450623:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572728.450627:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572728.450634:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.450639:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572728.450643:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572728.450648:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.450652:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572728.450658:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788572728.450662:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572728.450669:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572728.450673:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572728.450675:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572728.450678:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572728.450682:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572728.450688:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788572728.450694:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572728.450704:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516483456/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572728.450723:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572728.450727:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572728.450734:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516483456/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572728.450751:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572728.450768:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13426) 00000100:00000001:0.0:1788572728.960958:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572728.960963:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572728.960969:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572728.960974:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572728.960978:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572728.960982:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572728.960985:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572728.960987:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572728.960991:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572729.984928:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572729.984938:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572729.984947:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572729.984958:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572729.984968:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572729.984972:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572729.984985:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572729.984995:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572729.985010:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572730.689049:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572730.689054:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572730.689059:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572731.008925:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572731.008934:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572731.008940:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572731.008950:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572731.008959:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572731.008965:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.008973:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572731.008977:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.008984:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572731.009031:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572731.009035:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572731.009037:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572731.009044:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572731.009050:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572731.009054:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.009064:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572731.009072:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572731.009074:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572732.033033:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572732.033037:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572732.033039:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572732.033043:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572732.033048:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572732.033051:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572732.033054:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572732.033058:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572732.033061:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572732.288875:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308092142, at 4308092160 00000100:00000010:0.0:1788572732.288893:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7a80. 00000100:00000001:0.0:1788572733.057120:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572733.057125:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572733.057127:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572733.057130:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572733.057132:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572733.057134:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572733.057618:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572733.057623:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572733.057626:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572733.568918:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572733.568931:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572733.568934:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572733.568939:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572733.568944:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572733.568953:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13421 00000400:00000200:0.0:1788572733.568965:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572733.568973:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572733.568976:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572733.568980:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.568985:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572733.568990:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569015:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572733.569019:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572733.569021:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572733.569024:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572733.569027:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572733.569029:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572733.569031:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572733.569033:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569035:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572733.569036:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569037:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572733.569039:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572733.569042:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569044:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572733.569046:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572733.569049:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572733.569051:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.569053:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572733.569055:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572733.569061:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788572733.569064:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572733.569068:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572733.569069:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.569072:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572733.569075:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572733.569080:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572733.569088:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788572733.569095:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572733.569104:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516483584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572733.569113:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572733.569117:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572733.569126:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516483584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572733.569148:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572733.569153:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572733.569157:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572733.569159:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572733.569162:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572733.569165:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572733.569169:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13421 00000400:00000200:0.0:1788572733.569177:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572733.569184:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572733.569190:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572733.569197:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569202:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572733.569207:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569210:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572733.569213:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572733.569214:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572733.569215:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572733.569220:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572733.569225:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572733.569228:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572733.569233:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569239:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572733.569244:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569247:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572733.569250:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572733.569256:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569259:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572733.569270:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572733.569277:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572733.569280:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.569282:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572733.569288:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572733.569293:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.569296:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572733.569313:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788572733.569317:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572733.569320:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572733.569321:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.569326:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572733.569329:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572733.569332:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572733.569339:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572733.569344:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572733.569353:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516483712/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572733.569364:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572733.569367:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572733.569373:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516483712/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572733.569383:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572733.569389:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572733.569392:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572733.569394:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572733.569397:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572733.569399:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572733.569404:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13421 00000400:00000200:0.0:1788572733.569408:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572733.569413:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572733.569418:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572733.569427:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569428:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572733.569434:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569438:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572733.569440:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572733.569441:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572733.569443:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572733.569447:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572733.569450:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572733.569452:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572733.569456:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569459:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572733.569460:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569462:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572733.569464:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572733.569473:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569478:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572733.569480:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572733.569484:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572733.569487:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.569493:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572733.569497:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572733.569501:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.569502:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572733.569505:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788572733.569507:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572733.569510:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572733.569511:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.569512:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572733.569513:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572733.569516:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572733.569521:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572733.569534:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572733.569543:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516483840/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572733.569553:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572733.569555:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572733.569561:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516483840/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572733.569572:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572733.569578:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572733.569581:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572733.569583:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572733.569584:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572733.569586:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572733.569589:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13421 00000400:00000200:0.0:1788572733.569601:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572733.569607:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572733.569613:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572733.569618:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569622:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572733.569626:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569628:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572733.569634:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572733.569637:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572733.569641:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572733.569644:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572733.569648:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572733.569650:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572733.569656:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569661:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572733.569669:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569672:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572733.569675:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572733.569682:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569686:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572733.569688:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572733.569692:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572733.569699:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.569704:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572733.569707:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572733.569713:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.569716:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572733.569725:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788572733.569733:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572733.569741:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572733.569746:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.569757:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572733.569760:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572733.569765:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572733.569772:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788572733.569777:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572733.569785:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516483968/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572733.569800:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572733.569802:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572733.569841:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516483968/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572733.569857:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572733.569863:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572733.569867:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572733.569869:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572733.569879:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572733.569884:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572733.569890:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13421 00000400:00000200:0.0:1788572733.569900:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572733.569908:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572733.569913:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572733.569920:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569925:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572733.569931:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569937:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572733.569943:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572733.569947:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572733.569953:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572733.569957:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572733.569963:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572733.569968:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572733.569976:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569980:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572733.569989:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572733.569996:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572733.569999:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572733.570002:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572733.570009:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572733.570015:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572733.570019:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572733.570026:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.570031:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572733.570036:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572733.570041:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.570046:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572733.570061:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788572733.570066:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572733.570074:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572733.570078:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572733.570083:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572733.570086:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572733.570092:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572733.570099:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572733.570106:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572733.570117:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516484096/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572733.570135:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572733.570141:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572733.570149:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516484096/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572733.570171:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572733.570179:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13431) 00000100:00000001:0.0:1788572734.080916:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572734.080925:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572734.080930:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572734.080937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572734.080944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572734.080952:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572734.080976:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572734.080981:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572734.080986:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572735.104921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572735.104928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572735.104931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572735.104938:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572735.104944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572735.104947:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572735.104951:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572735.104954:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572735.104957:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572735.105168:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572735.105173:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572735.105175:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572735.105178:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572735.105181:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572735.105183:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572735.105184:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572735.105185:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572735.105187:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572735.809013:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572735.809020:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572735.809025:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572737.153043:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572737.153050:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572737.153059:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572737.153065:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572737.153070:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572737.153073:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572737.153079:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572737.153084:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572737.153095:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572737.408955:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308097261, at 4308097280 00000100:00000010:0.0:1788572737.408969:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7180. 00000100:00000001:0.0:1788572738.177066:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572738.177070:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572738.177071:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572738.177074:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572738.177078:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572738.177080:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572738.177081:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572738.177083:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572738.177084:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572738.688889:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572738.688899:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572738.688901:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572738.688904:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572738.688923:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572738.688932:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13426 00000400:00000200:0.0:1788572738.688942:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572738.688949:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572738.688953:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572738.688957:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.688961:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572738.688964:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.688967:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572738.688971:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572738.688973:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572738.688976:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572738.688979:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572738.688981:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572738.688984:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572738.688987:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.688989:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572738.688991:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.688994:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572738.688997:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572738.689000:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689002:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572738.689005:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572738.689009:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572738.689012:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.689015:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572738.689017:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572738.689024:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788572738.689028:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572738.689032:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572738.689035:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.689045:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572738.689048:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572738.689053:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572738.689060:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572738.689068:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572738.689078:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516484224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572738.689089:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572738.689093:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572738.689099:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516484224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572738.689123:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572738.689127:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572738.689130:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572738.689132:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572738.689134:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572738.689137:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572738.689140:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13426 00000400:00000200:0.0:1788572738.689145:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572738.689148:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572738.689151:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572738.689154:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689157:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572738.689160:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689163:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572738.689165:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572738.689167:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572738.689169:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572738.689171:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572738.689190:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572738.689193:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572738.689208:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689214:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572738.689218:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689223:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572738.689229:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572738.689235:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689240:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572738.689245:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572738.689253:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572738.689258:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.689263:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572738.689270:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572738.689277:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.689281:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572738.689287:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788572738.689292:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572738.689297:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572738.689300:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.689302:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572738.689304:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572738.689309:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572738.689323:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788572738.689330:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572738.689338:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516484352/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572738.689349:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572738.689353:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572738.689361:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516484352/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572738.689378:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572738.689383:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572738.689397:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572738.689402:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572738.689404:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572738.689411:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572738.689423:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13426 00000400:00000200:0.0:1788572738.689495:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572738.689503:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572738.689507:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572738.689513:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689516:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572738.689522:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689527:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572738.689532:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572738.689535:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572738.689539:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572738.689544:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572738.689551:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572738.689556:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572738.689562:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689566:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572738.689570:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689574:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572738.689580:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572738.689587:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689590:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572738.689595:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572738.689601:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572738.689605:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.689614:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572738.689618:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572738.689629:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.689633:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572738.689640:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788572738.689647:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572738.689655:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572738.689658:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.689661:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572738.689663:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572738.689667:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572738.689673:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788572738.689678:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572738.689688:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516484480/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572738.689707:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572738.689711:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572738.689720:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516484480/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572738.689746:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572738.689751:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572738.689756:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572738.689760:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572738.689763:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572738.689770:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572738.689776:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13426 00000400:00000200:0.0:1788572738.689786:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572738.689794:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572738.689799:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572738.689805:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689837:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572738.689840:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689852:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572738.689857:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572738.689860:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572738.689865:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572738.689869:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572738.689874:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572738.689878:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572738.689884:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689889:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572738.689894:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689899:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572738.689904:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572738.689909:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572738.689914:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572738.689919:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572738.689925:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572738.689930:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.689933:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572738.689938:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572738.689943:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.689948:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572738.689955:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788572738.689960:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572738.689964:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572738.689968:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.689974:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572738.689977:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572738.689981:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572738.689991:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572738.689996:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572738.690006:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516484608/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572738.690039:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572738.690044:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572738.690054:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516484608/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572738.690072:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572738.690077:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572738.690085:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572738.690090:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572738.690094:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572738.690100:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572738.690106:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13426 00000400:00000200:0.0:1788572738.690115:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572738.690122:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572738.690127:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572738.690134:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.690139:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572738.690145:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.690150:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572738.690155:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572738.690160:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572738.690164:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572738.690169:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572738.690187:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572738.690194:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572738.690203:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.690206:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572738.690209:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572738.690214:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572738.690225:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572738.690229:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572738.690235:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572738.690239:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572738.690244:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572738.690250:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.690255:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572738.690260:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572738.690266:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.690271:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572738.690277:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788572738.690283:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572738.690289:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572738.690292:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572738.690297:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572738.690300:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572738.690304:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572738.690312:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572738.690318:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572738.690328:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516484736/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572738.690343:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572738.690349:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572738.690356:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516484736/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572738.690376:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572738.690384:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13436) 00000100:00000001:0.0:1788572739.200941:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572739.200948:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572739.200952:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572739.200959:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572739.200964:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572739.200982:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572739.200990:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572739.200995:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572739.201002:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572740.929048:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572740.929054:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572740.929058:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572741.249003:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572741.249011:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572741.249015:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572741.249022:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572741.249029:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572741.249038:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572741.249073:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572741.249078:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572741.249081:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572742.272980:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572742.272985:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572742.272990:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572742.272998:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572742.273004:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572742.273007:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572742.273011:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572742.273013:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572742.273018:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572742.529060:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308102381, at 4308102400 00000100:00000010:0.0:1788572742.529069:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674a80. 00000100:00000001:0.0:1788572743.296899:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572743.296904:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572743.296908:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572743.296913:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572743.296918:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572743.296922:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572743.296929:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572743.296933:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572743.296937:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572743.808960:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572743.808996:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572743.808999:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572743.809003:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572743.809008:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572743.809016:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13431 00000400:00000200:0.0:1788572743.809026:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572743.809039:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572743.809042:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572743.809058:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809064:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572743.809067:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809069:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572743.809073:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572743.809076:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572743.809078:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572743.809082:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572743.809083:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572743.809085:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572743.809088:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809089:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572743.809090:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809091:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572743.809094:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572743.809096:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809098:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572743.809101:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572743.809104:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572743.809107:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809110:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572743.809112:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572743.809121:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788572743.809132:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572743.809137:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572743.809139:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809143:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572743.809146:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572743.809153:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572743.809161:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788572743.809169:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572743.809179:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516484864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572743.809188:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572743.809192:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572743.809196:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516484864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572743.809218:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572743.809222:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572743.809226:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572743.809228:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572743.809229:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572743.809232:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572743.809235:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13431 00000400:00000200:0.0:1788572743.809240:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572743.809243:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572743.809246:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572743.809249:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809251:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572743.809253:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809255:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572743.809258:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572743.809259:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572743.809261:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572743.809269:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572743.809272:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572743.809274:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572743.809278:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809280:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572743.809282:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809285:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572743.809287:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572743.809290:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809293:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572743.809294:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572743.809301:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572743.809304:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809306:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572743.809311:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572743.809314:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809316:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572743.809319:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788572743.809322:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572743.809324:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572743.809326:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809327:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572743.809329:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572743.809331:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572743.809335:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788572743.809339:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572743.809343:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516484992/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572743.809351:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572743.809353:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572743.809357:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516484992/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572743.809381:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572743.809384:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572743.809388:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572743.809390:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572743.809392:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572743.809394:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572743.809397:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13431 00000400:00000200:0.0:1788572743.809401:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572743.809404:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572743.809407:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572743.809410:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809412:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572743.809415:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809417:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572743.809419:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572743.809420:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572743.809422:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572743.809424:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572743.809426:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572743.809428:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572743.809431:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809433:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572743.809436:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809438:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572743.809441:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572743.809444:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809447:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572743.809453:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572743.809456:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572743.809458:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809461:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572743.809463:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572743.809466:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809468:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572743.809471:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788572743.809475:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572743.809477:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572743.809479:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809480:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572743.809482:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572743.809484:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572743.809488:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788572743.809491:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572743.809496:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516485120/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572743.809504:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572743.809507:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572743.809520:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516485120/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572743.809528:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572743.809531:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572743.809535:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572743.809536:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572743.809538:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572743.809541:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572743.809544:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13431 00000400:00000200:0.0:1788572743.809548:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572743.809552:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572743.809554:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572743.809567:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809569:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572743.809572:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809574:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572743.809577:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572743.809578:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572743.809580:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572743.809582:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572743.809584:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572743.809586:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572743.809589:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809592:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572743.809593:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809596:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572743.809599:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572743.809601:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809603:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572743.809605:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572743.809608:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572743.809610:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809612:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572743.809615:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572743.809618:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809619:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572743.809623:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788572743.809625:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572743.809628:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572743.809630:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809631:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572743.809633:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572743.809635:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572743.809639:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788572743.809646:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572743.809651:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516485248/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572743.809659:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572743.809661:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572743.809665:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516485248/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572743.809673:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572743.809677:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572743.809681:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572743.809682:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572743.809685:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572743.809688:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572743.809690:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13431 00000400:00000200:0.0:1788572743.809695:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572743.809698:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572743.809706:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572743.809709:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809712:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572743.809714:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809717:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572743.809719:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572743.809720:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572743.809722:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572743.809724:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572743.809727:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572743.809729:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572743.809733:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809735:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572743.809741:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809743:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572743.809746:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572743.809750:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572743.809752:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572743.809754:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572743.809757:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572743.809759:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809761:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572743.809764:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572743.809767:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809769:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572743.809773:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788572743.809776:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572743.809778:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572743.809780:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572743.809782:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572743.809784:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572743.809786:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572743.809791:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788572743.809794:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572743.809798:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516485376/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572743.809806:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572743.809844:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572743.809849:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516485376/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572743.809857:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572743.809862:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13441) 00000100:00000001:0.0:1788572744.320938:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572744.320945:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572744.320949:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572744.320973:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572744.320975:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572744.320977:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572744.320999:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572744.321001:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572744.321003:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572745.344978:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572745.344992:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572745.344996:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572745.345006:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572745.345035:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572745.345047:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572745.345078:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572745.345083:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572745.345087:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572745.345139:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572745.345144:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572745.345146:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572745.345150:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572745.345153:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572745.345156:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572745.345178:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572745.345183:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572745.345187:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572746.049005:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572746.049009:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572746.049013:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572747.392979:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572747.392987:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572747.392992:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572747.392999:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572747.393007:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572747.393013:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572747.393019:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572747.393024:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572747.393028:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572747.393061:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572747.393065:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572747.393076:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572747.393081:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572747.393085:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572747.393090:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572747.393100:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572747.393104:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572747.393107:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572747.648929:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308107501, at 4308107520 00000100:00000010:0.0:1788572747.648948:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674e40. 00000100:00000001:0.0:1788572748.416933:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572748.416938:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572748.416941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572748.416945:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572748.416949:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572748.416952:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572748.416969:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572748.416972:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572748.416977:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572748.928918:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572748.928928:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572748.928931:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572748.928935:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572748.928940:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572748.928947:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13436 00000400:00000200:0.0:1788572748.928955:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572748.928977:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572748.928980:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572748.928983:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.928986:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572748.928988:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.928990:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572748.928993:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572748.928995:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572748.929015:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572748.929018:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572748.929021:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572748.929023:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572748.929027:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929029:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572748.929030:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929032:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572748.929035:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572748.929038:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929041:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572748.929043:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572748.929046:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572748.929049:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.929051:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572748.929053:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572748.929059:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788572748.929064:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572748.929068:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572748.929069:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.929072:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572748.929075:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572748.929079:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572748.929087:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788572748.929094:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572748.929104:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516485504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572748.929115:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572748.929119:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572748.929124:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516485504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572748.929149:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572748.929158:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572748.929162:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572748.929164:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572748.929166:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572748.929169:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572748.929174:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13436 00000400:00000200:0.0:1788572748.929180:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572748.929184:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572748.929189:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572748.929196:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929202:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572748.929206:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929212:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572748.929218:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572748.929221:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572748.929225:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572748.929229:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572748.929235:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572748.929238:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572748.929241:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929242:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572748.929243:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929244:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572748.929248:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572748.929252:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929255:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572748.929256:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572748.929261:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572748.929268:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.929277:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572748.929289:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572748.929298:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.929302:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572748.929309:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788572748.929315:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572748.929320:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572748.929321:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.929323:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572748.929325:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572748.929328:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572748.929336:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788572748.929343:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572748.929357:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516485632/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572748.929387:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572748.929393:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572748.929405:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516485632/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572748.929423:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572748.929429:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572748.929434:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572748.929438:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572748.929441:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572748.929448:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572748.929456:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13436 00000400:00000200:0.0:1788572748.929481:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572748.929491:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572748.929495:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572748.929499:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929503:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572748.929505:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929519:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572748.929526:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572748.929531:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572748.929536:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572748.929540:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572748.929545:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572748.929550:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572748.929559:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929564:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572748.929569:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929574:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572748.929580:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572748.929587:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929593:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572748.929597:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572748.929603:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572748.929607:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.929609:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572748.929614:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572748.929618:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.929623:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572748.929630:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788572748.929635:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572748.929642:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572748.929645:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.929648:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572748.929650:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572748.929654:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572748.929662:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572748.929669:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572748.929678:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516485760/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572748.929714:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572748.929719:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572748.929728:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516485760/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572748.929736:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572748.929741:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572748.929748:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572748.929752:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572748.929755:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572748.929760:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572748.929765:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13436 00000400:00000200:0.0:1788572748.929774:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572748.929783:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572748.929788:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572748.929794:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929798:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572748.929803:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929826:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572748.929829:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572748.929830:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572748.929831:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572748.929833:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572748.929835:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572748.929837:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572748.929841:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929846:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572748.929851:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929856:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572748.929866:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572748.929872:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572748.929878:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572748.929883:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572748.929887:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572748.929893:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.929897:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572748.929901:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572748.929905:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.929908:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572748.929913:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788572748.929918:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572748.929921:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572748.929925:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.929930:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572748.929932:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572748.929935:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572748.929941:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572748.929945:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572748.929955:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516485888/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572748.929977:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572748.929981:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572748.929992:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516485888/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572748.930006:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572748.930012:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572748.930021:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572748.930024:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572748.930027:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572748.930031:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572748.930044:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13436 00000400:00000200:0.0:1788572748.930052:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572748.930060:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572748.930066:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572748.930076:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.930080:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572748.930087:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.930093:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572748.930097:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572748.930099:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572748.930102:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572748.930104:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572748.930105:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572748.930106:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572748.930109:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.930110:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572748.930113:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572748.930117:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572748.930121:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572748.930126:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572748.930130:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572748.930133:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572748.930136:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572748.930138:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.930139:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572748.930141:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572748.930145:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.930149:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572748.930154:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788572748.930159:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572748.930165:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572748.930174:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572748.930177:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572748.930180:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572748.930183:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572748.930190:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572748.930195:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572748.930204:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516486016/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572748.930235:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572748.930240:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572748.930249:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516486016/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572748.930271:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572748.930279:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13446) 00000100:00000001:0.0:1788572749.440906:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572749.440911:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572749.440915:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572749.440919:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572749.440923:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572749.440928:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572749.440947:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572749.440950:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572749.440952:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572749.441033:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572749.441036:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572749.441038:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572749.441041:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572749.441044:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572749.441046:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572749.441049:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572749.441052:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572749.441054:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572750.464952:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572750.464973:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572750.464979:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572750.464988:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572750.465023:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572750.465028:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572750.465038:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572750.465046:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572750.465051:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572751.169080:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572751.169083:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572751.169086:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572751.488910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572751.488915:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572751.488921:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572751.488930:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572751.488937:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572751.488944:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572751.488969:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572751.488971:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572751.488973:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572752.512993:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572752.513000:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572752.513005:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572752.513021:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572752.513033:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572752.513046:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572752.513059:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572752.513068:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572752.513083:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572752.768927:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308112621, at 4308112640 00000100:00000010:0.0:1788572752.768946:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674ba0. 00000100:00000001:0.0:1788572754.560920:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572754.560927:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572754.560933:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572754.560941:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572754.560948:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572754.560951:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572754.561005:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572754.561009:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572754.561013:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572755.584994:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572755.585003:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572755.585015:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572755.585031:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572755.585037:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572755.585047:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572755.585057:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572755.585062:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572755.585070:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572756.289159:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572756.289164:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572756.289168:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572756.608955:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572756.608959:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572756.608962:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572756.608965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572756.608980:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572756.608982:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572756.609009:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572756.609011:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572756.609015:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572756.609034:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572756.609042:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572756.609046:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572756.609051:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572756.609061:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572756.609066:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572756.609072:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572756.609076:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572756.609080:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572757.632959:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572757.632964:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572757.632983:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572757.632991:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572757.632997:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572757.633001:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572757.633020:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572757.633024:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572757.633037:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572757.888889:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308117742, at 4308117760 00000100:00000010:0.0:1788572757.888903:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674cc0. 00000100:00000001:0.0:1788572758.656951:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572758.656956:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572758.656960:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572758.656964:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572758.656986:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572758.656989:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572758.656993:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572758.656996:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572758.657000:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572759.680982:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572759.680991:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572759.680994:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572759.681001:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572759.681006:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572759.681011:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572759.681021:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572759.681024:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572759.681031:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572759.681068:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572759.681074:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572759.681081:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572759.681092:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572759.681096:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572759.681099:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572759.681121:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572759.681124:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572759.681126:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572760.704919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572760.704927:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572760.704931:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572760.704938:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572760.704944:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572760.704949:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572760.704957:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572760.704992:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572760.704997:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572761.408982:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572761.408985:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572761.408988:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572761.728950:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572761.728962:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572761.728966:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572761.728988:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572761.728994:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572761.728999:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572761.729016:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572761.729022:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572761.729029:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572762.752896:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572762.752903:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572762.752907:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572762.752916:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572762.752924:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572762.752930:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572762.752938:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572762.752943:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572762.752948:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572763.008895:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308122861, at 4308122880 00000100:00000010:0.0:1788572763.008912:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674240. 00000100:00000001:0.0:1788572763.776962:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572763.776969:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572763.776973:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572763.776994:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572763.777002:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572763.777007:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572763.777021:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572763.777026:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572763.777030:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572763.777049:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572763.777052:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572763.777053:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572763.777062:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572763.777065:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572763.777067:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572763.777068:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572763.777069:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572763.777072:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.800971:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572764.800998:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572764.801004:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572764.801015:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572764.801027:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572764.801033:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572764.801058:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572764.801063:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572764.801067:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572764.801187:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572764.801192:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572764.801194:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572764.801198:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572764.801201:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572764.801204:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572764.801206:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572764.801208:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572764.801210:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572765.825032:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572765.825046:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572765.825053:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572765.825062:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572765.825071:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572765.825077:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572765.825084:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572765.825091:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572765.825097:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572766.529099:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572766.529104:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572766.529108:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572766.848942:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572766.848951:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572766.848991:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572766.849008:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572766.849028:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572766.849036:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572766.849051:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572766.849056:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572766.849060:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572767.360964:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788572767.360973:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788572767.360978:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788572767.361218:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572767.361236:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572767.361242:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572767.361247:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572767.872946:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572767.872957:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572767.872966:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572767.872979:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572767.872998:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572767.873009:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572767.873018:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572767.873022:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572767.873027:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572768.128975:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308127981, at 4308128000 00000100:00000010:0.0:1788572768.129009:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea73c0. 00000100:00000001:0.0:1788572768.896913:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572768.896921:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572768.896926:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572768.896932:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572768.896937:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572768.896942:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572768.896947:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572768.896952:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572768.896957:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572769.920944:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572769.920957:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572769.920983:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572769.921003:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572769.921007:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572769.921012:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572769.921045:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572769.921049:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572769.921054:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572770.944936:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572770.944951:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572770.944954:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572770.944964:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572770.944979:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572770.945006:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572770.945024:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572770.945028:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572770.945039:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572771.649031:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572771.649036:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572771.649041:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572771.968965:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572771.968973:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572771.968977:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572771.968983:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572771.968999:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572771.969003:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572771.969009:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572771.969014:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572771.969019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572772.992951:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572772.992962:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572772.992968:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572772.992979:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572772.992987:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572772.993004:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572772.993020:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572772.993024:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572772.993027:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572773.248902:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308133101, at 4308133120 00000100:00000010:0.0:1788572773.248939:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7b40. 00000100:00000001:0.0:1788572774.016946:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572774.016954:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572774.016956:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572774.016967:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572774.016983:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572774.017005:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572774.017023:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572774.017031:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572774.017043:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572775.040965:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572775.040974:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572775.040978:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572775.040983:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572775.040989:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572775.041009:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572775.041024:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572775.041029:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572775.041032:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572776.064916:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572776.064924:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572776.064929:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572776.064937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572776.064947:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572776.064953:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572776.064961:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572776.064965:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572776.064971:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572776.065025:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572776.065030:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572776.065034:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572776.065037:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572776.065042:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572776.065044:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572776.065045:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572776.065047:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572776.065049:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572776.769023:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572776.769027:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572776.769030:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572777.088962:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572777.088969:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572777.088972:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572777.088980:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572777.088988:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572777.088992:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572777.089008:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572777.089013:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572777.089020:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572777.089063:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572777.089077:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572777.089090:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572777.089107:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572777.089127:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572777.089140:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572777.089158:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572777.089161:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572777.089166:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572778.112927:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572778.112934:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572778.112937:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572778.112942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572778.112949:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572778.112953:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572778.112955:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572778.112959:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572778.112963:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572778.368960:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308138222, at 4308138240 00000100:00000010:0.0:1788572778.368977:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ea0. 00000100:00000001:0.0:1788572779.137048:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572779.137061:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572779.137064:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572779.137069:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572779.137071:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572779.137084:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572779.137087:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572779.137088:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572779.137091:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572780.160949:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572780.160958:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572780.160962:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572780.160970:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572780.160978:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572780.160982:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572780.160987:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572780.160989:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572780.160993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572781.185070:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572781.185081:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572781.185087:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572781.185096:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572781.185104:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572781.185112:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572781.185118:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572781.185123:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572781.185128:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572781.889064:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572781.889070:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572781.889074:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572782.208911:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572782.208919:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572782.208923:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572782.208930:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572782.208937:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572782.208943:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572782.208957:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572782.208962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572782.208968:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572783.232912:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572783.232920:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572783.232924:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572783.232932:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572783.232960:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572783.232963:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572783.232968:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572783.232970:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572783.232973:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572783.233161:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572783.233167:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572783.233168:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572783.233172:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572783.233175:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572783.233178:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572783.233180:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572783.233182:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572783.233185:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572783.488917:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308143341, at 4308143360 00000100:00000010:0.0:1788572783.488936:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ba0. 00000100:00080000:0.0:1788572784.768915:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572784.768926:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572784.768929:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572784.768934:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572784.768940:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572784.768948:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13472 00000400:00000200:0.0:1788572784.768959:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572784.768967:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572784.768970:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572784.768973:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.768976:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572784.768978:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.768980:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572784.768983:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572784.768985:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572784.768987:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572784.768989:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572784.769028:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572784.769031:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572784.769034:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769035:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572784.769037:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769039:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572784.769041:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572784.769043:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769045:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572784.769047:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572784.769050:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572784.769052:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769053:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572784.769055:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572784.769060:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3100. 00000020:00000040:0.0:1788572784.769063:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572784.769066:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572784.769068:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769070:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572784.769072:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572784.769076:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572784.769082:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572784.769087:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572784.769095:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3100 x1875440516489984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572784.769103:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572784.769107:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572784.769111:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053a4b590] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3100 x1875440516489984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572784.769136:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572784.769139:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572784.769146:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572784.769148:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572784.769149:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572784.769151:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572784.769153:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13472 00000400:00000200:0.0:1788572784.769156:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572784.769160:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572784.769162:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572784.769164:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769166:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572784.769168:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769171:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572784.769175:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572784.769177:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572784.769181:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572784.769183:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572784.769186:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572784.769189:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572784.769192:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769194:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572784.769195:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769196:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572784.769198:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572784.769203:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769208:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572784.769213:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572784.769220:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572784.769223:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769224:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572784.769228:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572784.769239:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769243:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572784.769248:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788572784.769252:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572784.769255:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572784.769258:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769264:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572784.769266:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572784.769270:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572784.769281:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572784.769288:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572784.769300:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516490112/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572784.769314:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572784.769318:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572784.769327:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516490112/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572784.769342:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572784.769347:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572784.769355:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572784.769360:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572784.769365:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572784.769371:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572784.769378:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13472 00000400:00000200:0.0:1788572784.769389:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572784.769399:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572784.769406:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572784.769414:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769420:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572784.769422:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769432:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572784.769435:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572784.769436:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572784.769439:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572784.769442:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572784.769445:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572784.769448:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572784.769455:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769459:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572784.769460:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769465:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572784.769468:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572784.769473:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769478:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572784.769482:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572784.769486:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572784.769489:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769490:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572784.769492:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572784.769496:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769498:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572784.769504:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788572784.769509:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572784.769514:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572784.769516:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769519:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572784.769522:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572784.769524:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572784.769527:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572784.769534:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572784.769542:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516490240/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572784.769562:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572784.769564:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572784.769567:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516490240/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572784.769576:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572784.769578:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572784.769583:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572784.769585:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572784.769588:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572784.769593:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572784.769599:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13472 00000400:00000200:0.0:1788572784.769604:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572784.769610:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572784.769616:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572784.769622:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769626:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572784.769631:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769634:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572784.769636:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572784.769640:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572784.769642:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572784.769646:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572784.769650:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572784.769656:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572784.769659:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769664:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572784.769665:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769667:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572784.769673:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572784.769676:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769680:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572784.769682:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572784.769685:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572784.769691:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769692:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572784.769694:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572784.769699:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769704:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572784.769708:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788572784.769711:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572784.769715:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572784.769717:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769721:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572784.769724:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572784.769727:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572784.769732:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788572784.769736:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572784.769741:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516490368/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572784.769749:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572784.769750:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572784.769755:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516490368/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572784.769774:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572784.769776:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572784.769778:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572784.769781:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572784.769784:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572784.769788:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572784.769796:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13472 00000400:00000200:0.0:1788572784.769803:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572784.769827:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572784.769829:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572784.769832:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769834:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572784.769836:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769838:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572784.769840:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572784.769841:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572784.769842:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572784.769843:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572784.769845:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572784.769846:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572784.769849:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769852:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572784.769856:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769859:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572784.769863:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572784.769869:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572784.769873:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572784.769877:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572784.769881:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572784.769884:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769889:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572784.769892:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572784.769898:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769900:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572784.769905:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788572784.769910:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572784.769914:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572784.769920:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572784.769926:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572784.769929:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572784.769932:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572784.769936:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788572784.769943:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572784.769954:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516490496/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572784.769979:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572784.769983:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572784.769994:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516490496/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572784.770022:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572784.770028:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13482) 00000100:00000001:0.0:1788572785.281119:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572785.281127:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572785.281131:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572785.281134:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572785.281138:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572785.281141:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572785.281147:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572785.281149:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572785.281151:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.304872:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572786.304882:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572786.304885:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572786.304891:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572786.304899:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572786.304906:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572786.304912:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572786.304917:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572786.304923:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572786.304955:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572786.304960:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572786.304964:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572786.304967:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572786.304989:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572786.304992:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572786.304995:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572786.304996:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572786.304998:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572787.009021:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572787.009025:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572787.009027:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572787.328957:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572787.328962:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572787.328966:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572787.328971:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572787.328975:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572787.328980:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572787.328992:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572787.328995:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572787.328998:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572788.352924:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572788.352929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572788.352934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572788.352939:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572788.352945:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572788.352957:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572788.352993:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572788.352997:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572788.353000:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572788.608909:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308148461, at 4308148480 00000100:00000010:0.0:1788572788.608928:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7120. 00000100:00000001:0.0:1788572789.376958:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572789.376966:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572789.376969:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572789.376976:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572789.376980:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572789.376985:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572789.376992:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572789.376996:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572789.377003:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572789.888911:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572789.888919:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572789.888922:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572789.888926:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572789.888931:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572789.888938:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13477 00000400:00000200:0.0:1788572789.888947:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572789.888953:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572789.888955:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572789.888958:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.888960:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572789.888963:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.888965:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572789.888968:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572789.888969:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572789.888971:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572789.888974:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572789.888975:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572789.888977:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572789.888979:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.888981:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572789.888982:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.888984:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572789.888985:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572789.888988:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572789.888989:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572789.888991:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572789.888994:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572789.888996:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.888997:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572789.888999:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572789.889038:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788572789.889043:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572789.889047:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572789.889049:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889052:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572789.889055:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572789.889059:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572789.889067:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572789.889074:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572789.889084:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516490624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572789.889093:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572789.889097:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572789.889101:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516490624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572789.889118:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572789.889120:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572789.889123:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572789.889124:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572789.889125:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572789.889127:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572789.889130:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13477 00000400:00000200:0.0:1788572789.889134:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572789.889137:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572789.889139:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572789.889144:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889149:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572789.889153:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889155:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572789.889158:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572789.889161:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572789.889169:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572789.889173:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572789.889175:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572789.889177:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572789.889181:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889183:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572789.889185:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889189:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572789.889194:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572789.889200:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889206:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572789.889209:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572789.889215:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572789.889220:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889222:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572789.889227:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572789.889230:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889235:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572789.889240:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788572789.889244:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572789.889245:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572789.889247:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889252:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572789.889253:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572789.889255:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572789.889260:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572789.889267:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572789.889275:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516490752/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572789.889289:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572789.889292:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572789.889298:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516490752/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572789.889316:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572789.889319:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572789.889324:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572789.889326:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572789.889329:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572789.889332:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572789.889336:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13477 00000400:00000200:0.0:1788572789.889344:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572789.889350:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572789.889355:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572789.889361:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889365:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572789.889368:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889373:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572789.889376:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572789.889378:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572789.889379:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572789.889380:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572789.889384:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572789.889388:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572789.889393:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889396:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572789.889399:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889402:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572789.889404:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572789.889409:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889414:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572789.889424:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572789.889428:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572789.889433:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889437:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572789.889439:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572789.889444:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889449:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572789.889453:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788572789.889459:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572789.889462:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572789.889464:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889467:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572789.889470:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572789.889472:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572789.889478:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572789.889483:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572789.889489:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516490880/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572789.889516:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572789.889519:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572789.889540:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516490880/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572789.889548:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572789.889553:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572789.889557:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572789.889560:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572789.889561:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572789.889564:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572789.889569:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13477 00000400:00000200:0.0:1788572789.889576:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572789.889582:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572789.889587:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572789.889598:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889603:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572789.889607:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889609:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572789.889610:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572789.889611:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572789.889614:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572789.889617:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572789.889619:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572789.889620:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572789.889622:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889626:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572789.889630:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889635:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572789.889637:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572789.889640:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889643:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572789.889646:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572789.889652:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572789.889658:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889664:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572789.889669:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572789.889675:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889681:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572789.889689:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788572789.889696:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572789.889703:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572789.889707:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889712:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572789.889716:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572789.889720:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572789.889727:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572789.889741:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572789.889753:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516491008/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572789.889780:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572789.889782:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572789.889788:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516491008/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572789.889797:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572789.889799:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572789.889805:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572789.889832:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572789.889836:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572789.889839:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572789.889845:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13477 00000400:00000200:0.0:1788572789.889854:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572789.889862:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572789.889866:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572789.889872:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889876:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572789.889881:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889885:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572789.889889:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572789.889892:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572789.889894:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572789.889897:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572789.889899:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572789.889902:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572789.889907:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889910:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572789.889919:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889921:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572789.889925:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572789.889930:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572789.889935:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572789.889939:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572789.889943:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572789.889947:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889951:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572789.889954:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572789.889958:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889960:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572789.889965:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788572789.889971:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572789.889974:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572789.889975:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572789.889976:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572789.889978:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572789.889981:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572789.889986:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788572789.889993:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572789.890000:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516491136/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572789.890027:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572789.890033:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572789.890038:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516491136/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572789.890051:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572789.890055:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13487) 00000100:00000001:0.0:1788572791.424915:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572791.424922:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572791.424929:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572791.424954:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572791.424961:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572791.424966:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572791.424972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572791.424974:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572791.424977:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572792.129087:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572792.129091:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572792.129095:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572792.449009:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572792.449036:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572792.449040:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572792.449048:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572792.449063:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572792.449068:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572792.449100:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572792.449104:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572792.449109:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572793.472988:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572793.472997:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572793.473007:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572793.473029:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572793.473037:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572793.473040:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572793.473043:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572793.473045:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572793.473049:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572793.728919:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308153581, at 4308153600 00000100:00000010:0.0:1788572793.728941:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f00. 00000100:00000001:0.0:1788572794.496975:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572794.496982:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572794.496986:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572794.496992:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572794.496998:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572794.497003:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572794.497033:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572794.497036:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572794.497064:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572795.520922:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572795.520928:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572795.520932:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572795.520935:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572795.520940:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572795.520945:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572795.520968:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572795.520971:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572795.520975:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572796.544973:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572796.544981:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572796.544987:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572796.544995:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572796.545002:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572796.545007:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572796.545013:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572796.545018:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572796.545086:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572797.249071:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572797.249075:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572797.249078:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572798.592964:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572798.592970:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572798.592973:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572798.592978:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572798.592981:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572798.592983:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572798.593002:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572798.593003:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572798.593005:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572798.593017:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572798.593021:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572798.593036:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572798.593038:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572798.593043:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572798.593046:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572798.593051:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572798.593061:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572798.593064:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572798.848899:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308158701, at 4308158720 00000100:00000010:0.0:1788572798.848919:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7720. 00000100:00000001:0.0:1788572799.616926:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572799.616933:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572799.616939:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572799.616949:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572799.616957:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572799.616963:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572799.616992:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572799.616996:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572799.616999:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572800.640953:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572800.640963:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572800.640969:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572800.640977:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572800.640986:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572800.640991:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572800.640995:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572800.640998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572800.641002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572800.641058:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572800.641061:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572800.641064:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572800.641075:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572800.641080:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572800.641082:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572800.641084:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572800.641085:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572800.641086:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572801.664928:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572801.664938:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572801.664942:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572801.664952:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572801.664962:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572801.664967:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572801.664990:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572801.664995:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572801.665001:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572802.368963:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572802.368966:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572802.368969:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572802.688938:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572802.688946:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572802.688950:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572802.688955:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572802.688972:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572802.688975:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572802.688980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572802.688983:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572802.688986:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572802.689013:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572802.689020:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572802.689023:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572802.689052:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572802.689056:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572802.689060:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572802.689061:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572802.689063:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572802.689064:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572803.712956:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572803.712959:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572803.712962:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572803.712968:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572803.712971:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572803.712975:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572803.712978:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572803.712981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572803.712984:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572803.968892:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308163821, at 4308163840 00000100:00000010:0.0:1788572803.968909:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7de0. 00000100:00000001:0.0:1788572804.736928:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572804.736937:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572804.736953:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572804.736962:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572804.736971:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572804.736980:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572804.737004:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572804.737008:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572804.737016:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572805.249214:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788572805.249224:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572805.249243:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572805.249255:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572805.249262:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572805.760898:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572805.760911:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572805.760916:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572805.760924:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572805.760937:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572805.760949:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572805.760967:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572805.760977:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572805.760983:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572806.784941:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572806.784948:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572806.784953:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572806.784960:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572806.784967:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572806.784971:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572806.784975:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572806.784977:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572806.784980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572807.489062:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572807.489066:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572807.489069:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572807.808952:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572807.808963:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572807.808968:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572807.808976:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572807.808981:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572807.809006:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572807.809026:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572807.809029:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572807.809050:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572808.832915:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572808.832922:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572808.832926:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572808.832932:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572808.832939:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572808.832942:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572808.832968:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572808.832970:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572808.832972:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572808.832991:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572808.832996:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572808.832998:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572808.833000:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572808.833003:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572808.833006:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572808.833009:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572808.833011:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572808.833013:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572809.088933:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308168941, at 4308168960 00000100:00000010:0.0:1788572809.088951:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea78a0. 00000100:00000001:0.0:1788572809.856920:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572809.856929:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572809.856934:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572809.856944:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572809.856953:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572809.856965:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572809.856986:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572809.856996:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572809.857001:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572810.368968:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572810.368983:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572810.368987:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572810.369016:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572810.369021:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572810.369029:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13497 00000400:00000200:0.0:1788572810.369054:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572810.369061:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572810.369064:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572810.369067:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369070:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572810.369072:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369074:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572810.369079:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572810.369082:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572810.369085:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572810.369089:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572810.369092:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572810.369094:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572810.369098:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369100:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572810.369102:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369104:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572810.369109:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572810.369113:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369116:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572810.369120:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572810.369124:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572810.369128:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369131:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572810.369134:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572810.369142:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788572810.369146:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572810.369150:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572810.369159:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369162:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572810.369165:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572810.369170:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572810.369178:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788572810.369186:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572810.369195:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516493184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572810.369208:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572810.369213:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572810.369219:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516493184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572810.369246:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572810.369250:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572810.369255:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572810.369256:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572810.369257:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572810.369260:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572810.369262:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13497 00000400:00000200:0.0:1788572810.369266:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572810.369270:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572810.369272:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572810.369275:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369276:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572810.369278:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369280:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572810.369282:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572810.369286:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572810.369288:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572810.369290:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572810.369293:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572810.369303:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572810.369311:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369316:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572810.369319:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369323:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572810.369326:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572810.369330:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369333:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572810.369337:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572810.369345:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572810.369350:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369352:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572810.369355:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572810.369358:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369360:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572810.369365:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788572810.369369:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572810.369373:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572810.369374:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369376:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572810.369380:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572810.369383:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572810.369390:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572810.369396:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572810.369403:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516493312/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572810.369414:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572810.369416:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572810.369419:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516493312/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572810.369427:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572810.369437:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572810.369442:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572810.369445:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572810.369447:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572810.369452:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572810.369458:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13497 00000400:00000200:0.0:1788572810.369467:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572810.369472:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572810.369475:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572810.369481:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369483:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572810.369489:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369492:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572810.369494:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572810.369495:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572810.369496:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572810.369497:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572810.369500:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572810.369504:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572810.369506:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369509:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572810.369514:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369517:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572810.369520:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572810.369524:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369528:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572810.369530:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572810.369534:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572810.369545:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369551:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572810.369558:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572810.369564:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369565:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572810.369568:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788572810.369570:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572810.369574:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572810.369576:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369581:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572810.369583:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572810.369586:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572810.369590:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572810.369595:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572810.369603:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516493440/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572810.369613:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572810.369617:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572810.369621:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516493440/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572810.369634:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572810.369637:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572810.369640:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572810.369641:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572810.369643:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572810.369648:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572810.369653:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13497 00000400:00000200:0.0:1788572810.369659:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572810.369663:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572810.369666:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572810.369668:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369674:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572810.369678:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369687:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572810.369690:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572810.369693:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572810.369695:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572810.369697:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572810.369700:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572810.369703:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572810.369710:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369714:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572810.369718:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369721:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572810.369724:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572810.369727:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369728:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572810.369729:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572810.369733:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572810.369737:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369739:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572810.369742:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572810.369746:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369750:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572810.369755:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788572810.369759:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572810.369762:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572810.369763:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369765:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572810.369766:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572810.369770:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572810.369774:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572810.369779:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572810.369788:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516493568/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572810.369828:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572810.369830:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572810.369834:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516493568/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572810.369840:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572810.369843:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572810.369848:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572810.369851:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572810.369853:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572810.369859:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572810.369862:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13497 00000400:00000200:0.0:1788572810.369868:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572810.369873:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572810.369876:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572810.369879:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369883:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572810.369887:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369892:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572810.369896:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572810.369899:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572810.369902:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572810.369904:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572810.369905:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572810.369909:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572810.369913:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369918:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572810.369921:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369926:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572810.369933:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572810.369940:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572810.369942:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572810.369946:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572810.369949:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572810.369953:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369956:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572810.369957:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572810.369960:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369965:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572810.369971:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788572810.369974:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572810.369978:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572810.369980:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.369981:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572810.369982:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572810.369983:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572810.369986:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788572810.369989:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572810.369998:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516493696/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572810.370005:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572810.370009:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572810.370017:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516493696/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572810.370028:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572810.370032:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13507) 00000100:00000001:0.0:1788572810.880971:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572810.880974:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572810.880977:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572810.880983:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572810.880989:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572810.881003:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572810.881007:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572810.881010:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572810.881012:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572810.881050:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572810.881053:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572810.881055:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572810.881058:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572810.881061:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572810.881062:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572810.881064:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572810.881065:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572810.881067:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572811.905003:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572811.905121:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572811.905125:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572811.905131:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572811.905135:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572811.905139:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572811.905143:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572811.905148:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572811.905157:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572812.609207:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572812.609212:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572812.609217:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572812.928950:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572812.928962:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572812.928966:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572812.928977:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572812.928986:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572812.928994:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572812.929005:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572812.929012:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572812.929020:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572812.929103:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572812.929109:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572812.929111:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572812.929115:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572812.929133:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572812.929136:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572812.929138:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572812.929140:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572812.929142:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572813.952992:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572813.953001:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572813.953004:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572813.953018:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572813.953031:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572813.953043:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572813.953064:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572813.953073:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572813.953099:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572814.208970:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308174061, at 4308174080 00000100:00000010:0.0:1788572814.208991:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7900. 00000100:00000001:0.0:1788572814.976935:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572814.976944:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572814.976948:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572814.976953:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572814.976961:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572814.976969:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572814.976978:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572814.976984:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572814.976996:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572816.000952:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572816.000960:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572816.000964:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572816.000971:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572816.000981:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572816.000987:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572816.000993:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572816.001002:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572816.001016:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572817.025004:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572817.025012:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572817.025017:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572817.025044:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572817.025049:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572817.025053:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572817.025057:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572817.025063:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572817.025072:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572817.729022:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572817.729027:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572817.729030:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572818.049003:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572818.049015:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572818.049025:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572818.049042:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572818.049052:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572818.049058:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572818.049074:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572818.049077:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572818.049080:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572819.072987:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572819.072995:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572819.073001:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572819.073007:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572819.073016:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572819.073021:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572819.073045:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572819.073054:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572819.073059:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572819.328903:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308179181, at 4308179200 00000100:00000010:0.0:1788572819.328919:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674000. 00000100:00000001:0.0:1788572820.096934:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572820.096942:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572820.096945:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572820.096951:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572820.096957:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572820.096964:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572820.096969:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572820.096974:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572820.096995:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572820.608961:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572820.608970:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572820.608973:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572820.608976:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572820.608980:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572820.608987:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13508 00000400:00000200:0.0:1788572820.608994:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572820.609000:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572820.609002:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572820.609005:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609007:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572820.609009:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609011:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572820.609014:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572820.609016:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572820.609018:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572820.609020:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572820.609022:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572820.609023:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572820.609025:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609027:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572820.609029:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609030:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572820.609032:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572820.609035:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609037:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572820.609038:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572820.609041:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572820.609043:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.609060:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572820.609061:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572820.609067:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788572820.609070:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572820.609073:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572820.609074:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.609076:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572820.609079:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572820.609082:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572820.609088:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572820.609093:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572820.609100:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516494464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572820.609108:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572820.609112:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572820.609115:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516494464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572820.609134:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572820.609137:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572820.609139:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572820.609140:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572820.609142:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572820.609143:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572820.609145:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13508 00000400:00000200:0.0:1788572820.609148:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572820.609151:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572820.609152:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572820.609155:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609158:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572820.609159:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609161:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572820.609169:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572820.609172:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572820.609177:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572820.609181:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572820.609186:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572820.609192:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572820.609203:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609210:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572820.609231:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609232:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572820.609236:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572820.609243:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609248:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572820.609250:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572820.609258:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572820.609264:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.609269:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572820.609275:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572820.609278:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.609283:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572820.609287:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788572820.609292:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572820.609295:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572820.609297:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.609299:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572820.609300:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572820.609302:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572820.609309:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572820.609314:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572820.609324:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516494592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572820.609333:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572820.609341:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572820.609347:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516494592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572820.609363:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572820.609367:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572820.609371:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572820.609373:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572820.609375:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572820.609377:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572820.609381:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13508 00000400:00000200:0.0:1788572820.609437:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572820.609443:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572820.609446:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572820.609448:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609450:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572820.609454:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609457:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572820.609458:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572820.609459:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572820.609461:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572820.609465:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572820.609469:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572820.609472:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572820.609477:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609480:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572820.609486:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609491:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572820.609499:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572820.609507:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609519:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572820.609533:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572820.609539:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572820.609543:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.609547:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572820.609551:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572820.609556:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.609564:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572820.609570:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788572820.609575:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572820.609581:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572820.609584:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.609589:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572820.609593:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572820.609598:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572820.609604:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788572820.609612:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572820.609621:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516494720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572820.609633:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572820.609638:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572820.609646:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516494720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572820.609656:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572820.609661:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572820.609667:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572820.609670:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572820.609673:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572820.609676:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572820.609680:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13508 00000400:00000200:0.0:1788572820.609689:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572820.609705:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572820.609711:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572820.609720:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609726:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572820.609733:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609737:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572820.609742:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572820.609744:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572820.609745:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572820.609747:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572820.609749:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572820.609750:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572820.609757:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609762:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572820.609765:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609769:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572820.609774:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572820.609780:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609783:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572820.609787:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572820.609792:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572820.609797:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.609803:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572820.609835:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572820.609841:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.609845:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572820.609852:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788572820.609856:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572820.609862:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572820.609865:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.609868:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572820.609877:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572820.609880:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572820.609886:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572820.609891:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572820.609899:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516494848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572820.609913:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572820.609919:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572820.609926:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516494848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572820.609939:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572820.609944:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572820.609952:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572820.609956:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572820.609958:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572820.609962:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572820.609965:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13508 00000400:00000200:0.0:1788572820.609968:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572820.609972:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572820.609977:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572820.609983:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609988:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572820.609992:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.609995:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572820.609998:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572820.610001:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572820.610006:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572820.610010:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572820.610015:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572820.610019:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572820.610030:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.610034:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572820.610038:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572820.610041:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572820.610044:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572820.610048:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572820.610052:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572820.610055:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572820.610059:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572820.610062:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.610066:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572820.610069:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572820.610074:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.610077:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572820.610084:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788572820.610089:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572820.610092:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572820.610095:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572820.610098:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572820.610101:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572820.610103:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572820.610109:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788572820.610112:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572820.610121:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516494976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572820.610137:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572820.610140:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572820.610148:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516494976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572820.610168:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572820.610176:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13518) 00000100:00000001:0.0:1788572821.120942:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572821.120956:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572821.120961:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572821.120967:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572821.120974:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572821.120977:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572821.121011:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572821.121016:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572821.121019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572822.144989:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572822.144999:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572822.145003:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572822.145016:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572822.145031:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572822.145041:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572822.145045:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572822.145048:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572822.145051:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572822.849054:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572822.849060:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572822.849065:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572823.168919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572823.168928:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572823.168931:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572823.168938:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572823.168944:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572823.168955:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572823.168960:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572823.168966:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572823.168974:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572824.193005:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572824.193012:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572824.193016:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572824.193022:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572824.193027:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572824.193033:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572824.193042:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572824.193047:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572824.193071:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572824.448875:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308184301, at 4308184320 00000100:00000010:0.0:1788572824.448890:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674060. 00000100:00000001:0.0:1788572825.216901:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572825.216910:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572825.216913:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572825.216919:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572825.216924:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572825.216929:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572825.216935:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572825.216940:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572825.216946:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.216991:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572825.216996:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572825.216998:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572825.217001:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572825.217005:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572825.217011:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572825.217016:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572825.217021:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572825.217027:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572825.728903:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572825.728912:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572825.728915:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572825.728918:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572825.728922:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572825.728929:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13513 00000400:00000200:0.0:1788572825.728938:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572825.728944:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572825.728947:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572825.728950:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.728953:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572825.728955:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.728957:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572825.728976:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572825.728978:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572825.728980:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572825.728983:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572825.728984:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572825.728993:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572825.728996:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.728998:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572825.728999:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729000:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572825.729003:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572825.729005:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729007:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572825.729009:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572825.729012:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572825.729014:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.729015:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572825.729017:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572825.729022:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788572825.729025:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572825.729028:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572825.729029:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.729032:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572825.729034:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572825.729038:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572825.729044:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788572825.729049:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572825.729072:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516495104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572825.729081:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572825.729085:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572825.729089:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516495104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572825.729108:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572825.729111:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572825.729114:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572825.729115:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572825.729116:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572825.729118:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572825.729120:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13513 00000400:00000200:0.0:1788572825.729123:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572825.729126:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572825.729128:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572825.729131:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729133:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572825.729135:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729136:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572825.729138:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572825.729140:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572825.729142:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572825.729146:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572825.729150:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572825.729154:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572825.729158:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729160:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572825.729165:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729170:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572825.729175:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572825.729180:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729183:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572825.729192:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572825.729200:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572825.729205:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.729207:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572825.729211:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572825.729215:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.729217:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572825.729223:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788572825.729227:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572825.729231:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572825.729232:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.729237:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572825.729240:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572825.729242:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572825.729246:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788572825.729251:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572825.729259:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516495232/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572825.729268:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572825.729272:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572825.729278:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516495232/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572825.729326:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572825.729330:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572825.729332:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572825.729333:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572825.729336:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572825.729339:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572825.729344:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13513 00000400:00000200:0.0:1788572825.729352:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572825.729357:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572825.729359:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572825.729369:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729373:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572825.729377:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729381:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572825.729385:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572825.729386:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572825.729389:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572825.729392:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572825.729396:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572825.729400:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572825.729414:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729418:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572825.729421:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729424:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572825.729429:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572825.729436:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729441:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572825.729445:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572825.729448:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572825.729452:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.729454:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572825.729456:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572825.729460:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.729469:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572825.729478:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788572825.729481:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572825.729483:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572825.729486:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.729488:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572825.729491:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572825.729493:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572825.729497:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788572825.729508:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572825.729516:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516495360/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572825.729534:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572825.729538:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572825.729544:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516495360/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572825.729565:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572825.729571:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572825.729576:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572825.729582:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572825.729587:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572825.729595:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572825.729602:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13513 00000400:00000200:0.0:1788572825.729611:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572825.729617:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572825.729622:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572825.729629:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729636:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572825.729642:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729648:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572825.729655:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572825.729660:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572825.729664:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572825.729669:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572825.729673:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572825.729678:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572825.729685:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729689:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572825.729702:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729709:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572825.729716:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572825.729725:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729733:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572825.729738:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572825.729746:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572825.729753:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.729760:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572825.729766:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572825.729772:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.729779:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572825.729787:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788572825.729794:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572825.729801:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572825.729802:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.729824:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572825.729826:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572825.729828:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572825.729832:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788572825.729835:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572825.729841:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516495488/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572825.729856:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572825.729861:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572825.729869:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516495488/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572825.729883:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572825.729887:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572825.729889:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572825.729892:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572825.729900:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572825.729904:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572825.729907:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13513 00000400:00000200:0.0:1788572825.729914:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572825.729919:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572825.729922:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572825.729926:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729928:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572825.729932:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729936:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572825.729940:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572825.729943:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572825.729945:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572825.729948:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572825.729951:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572825.729954:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572825.729961:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729965:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572825.729967:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729973:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572825.729975:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572825.729980:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572825.729986:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572825.729989:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572825.729992:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572825.729998:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.730002:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572825.730006:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572825.730012:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.730017:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572825.730028:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788572825.730032:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572825.730037:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572825.730038:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572825.730039:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572825.730040:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572825.730042:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572825.730045:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572825.730051:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572825.730058:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516495616/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572825.730070:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572825.730074:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572825.730081:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516495616/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572825.730096:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572825.730102:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13523) 00000100:00000001:0.0:1788572826.240918:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572826.240926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572826.240929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572826.240936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572826.240944:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572826.240950:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572826.240965:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572826.240969:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572826.240973:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572826.240987:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572826.240991:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572826.240996:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572826.241002:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572826.241009:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572826.241015:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572826.241020:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572826.241024:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572826.241027:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572827.264886:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572827.264893:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572827.264894:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572827.264899:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572827.264904:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572827.264909:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572827.264913:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572827.264916:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572827.264921:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572827.264956:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572827.264959:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572827.264963:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572827.264970:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572827.264975:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572827.264979:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572827.264982:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572827.264985:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572827.264988:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572827.969041:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572827.969044:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572827.969047:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572828.288955:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572828.288965:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572828.288969:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572828.288981:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572828.289000:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572828.289009:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572828.289028:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572828.289031:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572828.289035:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.312973:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572829.312978:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572829.312982:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572829.312989:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572829.312994:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572829.312998:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572829.313010:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572829.313013:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572829.313027:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572829.313059:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572829.313062:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572829.313063:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572829.313065:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572829.313067:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572829.313070:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572829.313072:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572829.313073:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572829.313075:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572829.568874:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308189421, at 4308189440 00000100:00000010:0.0:1788572829.568897:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674420. 00000100:00000001:0.0:1788572830.337044:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572830.337055:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572830.337061:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572830.337081:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572830.337087:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572830.337090:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572830.337094:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572830.337097:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572830.337100:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572831.360995:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572831.361009:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572831.361019:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572831.361035:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572831.361043:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572831.361049:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572831.361082:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572831.361088:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572831.361092:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572832.384984:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572832.384989:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572832.384992:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572832.384998:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572832.385011:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572832.385019:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572832.385036:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572832.385054:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572832.385069:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572833.089044:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572833.089048:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572833.089051:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572833.408927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572833.408933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572833.408938:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572833.408945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572833.408952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572833.408957:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572833.408968:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572833.408970:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572833.408976:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572834.688908:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308194542, at 4308194560 00000100:00000010:0.0:1788572834.688925:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea71e0. 00000100:00000001:0.0:1788572835.456987:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572835.457011:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572835.457018:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572835.457031:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572835.457042:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572835.457051:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572835.457065:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572835.457073:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572835.457084:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572836.480901:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572836.480909:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572836.480914:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572836.480920:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572836.480927:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572836.480932:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572836.480951:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572836.480954:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572836.480957:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572837.504940:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572837.504952:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572837.504956:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572837.504969:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572837.504976:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572837.504979:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572837.504993:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572837.504995:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572837.504998:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572838.209135:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572838.209141:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572838.209147:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572838.528905:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572838.528912:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572838.528915:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572838.528923:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572838.528928:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572838.528932:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572838.528938:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572838.528942:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572838.528946:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572839.808953:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308199661, at 4308199680 00000100:00000010:0.0:1788572839.808974:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7840. 00000100:00000001:0.0:1788572840.577101:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572840.577105:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572840.577106:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572840.577110:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572840.577112:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572840.577115:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572840.577117:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572840.577119:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572840.577121:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572841.600909:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572841.600917:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572841.600922:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572841.600929:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572841.600938:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572841.600945:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572841.600954:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572841.600958:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572841.600981:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572842.624923:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572842.624928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572842.624932:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572842.624937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572842.624941:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572842.624946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572842.624949:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572842.624953:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572842.624956:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572843.329059:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572843.329064:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572843.329068:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572843.648935:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572843.648945:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572843.648949:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572843.648954:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572843.648961:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572843.648967:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572843.648980:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572843.648984:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572843.648987:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572844.160947:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788572844.160956:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788572844.160962:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788572844.161126:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572844.161142:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572844.161151:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572844.161154:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572844.672934:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572844.672942:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572844.672945:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572844.672953:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572844.672965:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572844.672976:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572844.672983:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572844.672987:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572844.673010:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572844.928896:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308204781, at 4308204800 00000100:00000010:0.0:1788572844.928910:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f00. 00000100:00000001:0.0:1788572845.696941:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572845.696949:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572845.696953:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572845.696959:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572845.696967:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572845.696974:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572845.697001:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572845.697005:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572845.697009:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572847.744944:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572847.744952:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572847.744957:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572847.744966:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572847.744974:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572847.744986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572847.745023:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572847.745028:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572847.745042:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572848.449022:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572848.449027:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572848.449045:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572848.769205:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572848.769209:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572848.769211:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572848.769214:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572848.769217:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572848.769219:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572848.769221:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572848.769222:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572848.769225:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572848.769236:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572848.769240:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572848.769241:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572848.769244:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572848.769255:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572848.769259:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572848.769261:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572848.769264:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572848.769277:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572849.792904:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572849.792912:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572849.792915:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572849.792930:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572849.792935:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572849.792941:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572849.792965:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572849.792968:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572849.792971:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572850.048949:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308209902, at 4308209920 00000100:00000010:0.0:1788572850.048969:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7120. 00000100:00000001:0.0:1788572850.816944:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572850.816954:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572850.816959:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572850.816969:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572850.816977:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572850.816983:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572850.816989:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572850.816993:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572850.816997:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572850.817038:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572850.817042:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572850.817044:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572850.817046:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572850.817049:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572850.817052:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572850.817053:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572850.817055:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572850.817057:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572851.840914:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572851.840923:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572851.840926:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572851.840956:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572851.840963:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572851.840967:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572851.840976:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572851.840982:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572851.840988:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572852.864946:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572852.864955:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572852.864963:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572852.864974:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572852.864979:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572852.864984:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572852.864989:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572852.864995:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572852.865000:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572852.865038:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572852.865054:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572852.865056:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572852.865063:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572852.865067:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572852.865071:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572852.865075:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572852.865077:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572852.865080:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572853.569080:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572853.569084:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572853.569088:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572853.888944:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572853.888955:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572853.888960:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572853.888966:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572853.888975:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572853.888982:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572853.888992:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572853.888997:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572853.889001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572853.889138:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572853.889141:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572853.889156:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572853.889159:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572853.889163:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572853.889166:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572853.889169:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572853.889172:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572853.889175:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572854.912975:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572854.912986:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572854.912997:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572854.913016:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572854.913034:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572854.913045:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572854.913050:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572854.913056:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572854.913060:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572855.168940:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308215021, at 4308215040 00000100:00000010:0.0:1788572855.168962:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ba0. 00000100:00000001:0.0:1788572855.936906:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572855.936926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572855.936929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572855.936936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572855.936941:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572855.936946:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572855.936953:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572855.936956:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572855.936969:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572856.960959:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572856.960967:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572856.960973:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572856.960978:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572856.960981:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572856.960983:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572856.960996:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572856.961001:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572856.961005:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572857.984922:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572857.984945:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572857.984951:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572857.984971:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572857.984976:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572857.984981:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572857.984989:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572857.985001:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572857.985006:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572858.689053:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572858.689058:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572858.689061:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572859.008917:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572859.008924:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572859.008928:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572859.008935:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572859.008942:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572859.008947:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572859.008972:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572859.008989:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572859.008992:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572860.032881:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572860.032888:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572860.032892:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572860.032902:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572860.032909:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572860.032913:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572860.032920:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572860.032923:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572860.032927:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572860.032955:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572860.032960:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572860.032963:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572860.032967:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572860.032969:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572860.032971:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572860.032973:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572860.032974:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572860.032975:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572860.288884:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308220141, at 4308220160 00000100:00000010:0.0:1788572860.288902:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ea0. 00000100:00000001:0.0:1788572861.056917:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572861.056925:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572861.056931:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572861.056940:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572861.056948:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572861.056953:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572861.056971:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572861.056976:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572861.056982:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572862.080877:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572862.080885:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572862.080888:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572862.080895:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572862.080902:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572862.080909:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572862.080918:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572862.080923:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572862.080929:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572863.104919:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572863.104925:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572863.104929:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572863.104936:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572863.104942:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572863.104948:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572863.105855:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572863.105859:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572863.105863:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572863.809100:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572863.809103:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572863.809106:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572864.128931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572864.128942:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572864.128948:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572864.128957:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572864.128965:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572864.128994:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572864.129034:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572864.129039:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572864.129045:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572865.152949:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572865.152955:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572865.152958:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572865.152961:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572865.152965:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572865.152972:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572865.152990:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572865.152992:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572865.152994:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572865.153020:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572865.153025:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572865.153027:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572865.153030:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572865.153033:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572865.153036:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572865.153039:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572865.153041:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572865.153043:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572865.408966:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308225261, at 4308225280 00000100:00000010:0.0:1788572865.408988:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7b40. 00000100:00000001:0.0:1788572866.176936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572866.176941:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572866.176945:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572866.176949:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572866.176955:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572866.176958:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572866.176963:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572866.176966:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572866.176969:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572867.201017:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572867.201031:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572867.201041:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572867.201053:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572867.201067:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572867.201082:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572867.201086:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572867.201089:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572867.201092:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572868.224943:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572868.224949:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572868.224954:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572868.224961:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572868.224968:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572868.224974:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572868.225000:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572868.225004:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572868.225007:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572868.928972:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572868.928975:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572868.928977:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572869.248908:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572869.248916:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572869.248920:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572869.248929:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572869.248936:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572869.248942:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572869.248953:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572869.248958:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572869.248964:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572870.272923:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572870.272931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572870.272935:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572870.272941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572870.272945:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572870.272949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572870.272982:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572870.272986:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572870.272989:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572870.528889:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308230382, at 4308230400 00000100:00000010:0.0:1788572870.528910:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea73c0. 00000100:00000001:0.0:1788572871.296918:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572871.296940:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572871.296946:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572871.296951:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572871.296958:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572871.296964:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572871.296973:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572871.296977:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572871.296983:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572871.808950:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572871.808959:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572871.808962:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572871.808965:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572871.808969:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572871.808976:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13559 00000400:00000200:0.0:1788572871.808985:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572871.808991:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572871.808994:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572871.808996:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.808998:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572871.809000:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809002:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572871.809005:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572871.809007:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572871.809009:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572871.809011:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572871.809013:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572871.809015:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572871.809017:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809018:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572871.809020:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809021:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572871.809023:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572871.809043:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809045:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572871.809048:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572871.809050:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572871.809052:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809054:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572871.809056:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572871.809061:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788572871.809064:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572871.809067:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572871.809068:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809071:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572871.809073:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572871.809077:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572871.809083:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788572871.809089:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572871.809114:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516500864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572871.809126:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572871.809131:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572871.809136:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516500864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572871.809157:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572871.809161:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572871.809164:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572871.809165:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572871.809167:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572871.809169:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572871.809172:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13559 00000400:00000200:0.0:1788572871.809176:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572871.809179:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572871.809183:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572871.809193:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809196:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572871.809200:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809202:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572871.809206:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572871.809207:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572871.809211:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572871.809213:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572871.809217:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572871.809221:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572871.809240:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809243:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572871.809245:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809246:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572871.809248:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572871.809249:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809251:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572871.809253:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572871.809258:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572871.809261:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809266:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572871.809270:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572871.809274:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809279:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572871.809286:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788572871.809289:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572871.809295:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572871.809297:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809300:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572871.809301:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572871.809306:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572871.809310:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788572871.809324:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572871.809333:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516500992/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572871.809347:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572871.809351:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572871.809356:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516500992/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572871.809366:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572871.809370:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572871.809377:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572871.809379:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572871.809383:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572871.809386:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572871.809391:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13559 00000400:00000200:0.0:1788572871.809399:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572871.809403:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572871.809408:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572871.809415:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809418:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572871.809422:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809424:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572871.809428:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572871.809431:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572871.809432:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572871.809433:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572871.809435:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572871.809439:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572871.809445:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809446:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572871.809455:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809458:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572871.809462:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572871.809467:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809471:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572871.809475:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572871.809480:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572871.809484:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809488:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572871.809490:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572871.809494:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809498:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572871.809502:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788572871.809505:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572871.809507:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572871.809508:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809510:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572871.809512:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572871.809515:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572871.809519:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572871.809523:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572871.809530:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516501120/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572871.809541:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572871.809543:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572871.809548:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516501120/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572871.809557:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572871.809559:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572871.809571:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572871.809576:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572871.809584:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572871.809588:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572871.809593:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13559 00000400:00000200:0.0:1788572871.809599:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572871.809602:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572871.809605:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572871.809611:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809614:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572871.809619:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809624:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572871.809629:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572871.809633:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572871.809637:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572871.809642:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572871.809647:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572871.809654:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572871.809663:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809669:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572871.809676:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809682:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572871.809687:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572871.809694:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809698:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572871.809701:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572871.809705:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572871.809708:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809709:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572871.809712:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572871.809716:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809718:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572871.809729:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788572871.809735:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572871.809740:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572871.809743:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809745:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572871.809748:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572871.809750:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572871.809754:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788572871.809760:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572871.809768:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516501248/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572871.809781:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572871.809784:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572871.809792:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516501248/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572871.809798:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572871.809802:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572871.809831:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572871.809833:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572871.809836:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572871.809842:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572871.809846:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13559 00000400:00000200:0.0:1788572871.809854:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572871.809859:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572871.809863:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572871.809868:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809872:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572871.809876:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809880:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572871.809883:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572871.809891:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572871.809894:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572871.809897:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572871.809901:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572871.809905:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572871.809910:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809914:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572871.809918:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809922:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572871.809926:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572871.809931:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572871.809935:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572871.809939:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572871.809942:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572871.809948:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809952:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572871.809955:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572871.809960:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809963:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572871.809969:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788572871.809972:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572871.809978:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572871.809979:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572871.809984:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572871.809988:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572871.809991:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572871.809995:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572871.810000:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572871.810005:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516501376/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572871.810031:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572871.810035:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572871.810048:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516501376/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572871.810060:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572871.810066:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13569) 00000100:00000001:0.0:1788572872.320941:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572872.320950:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572872.320957:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572872.320967:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572872.320973:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572872.320977:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572872.320994:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572872.320997:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572872.321000:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.344910:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572873.344917:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572873.344921:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572873.344927:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572873.344942:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572873.344950:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572873.344953:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572873.344957:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572873.344961:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572873.345031:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572873.345035:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572873.345036:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572873.345039:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572873.345042:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572873.345044:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572873.345046:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572873.345047:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572873.345050:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572874.049034:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572874.049039:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572874.049043:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572875.648920:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308235501, at 4308235520 00000100:00000010:0.0:1788572875.648938:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7540. 00000100:00000001:0.0:1788572876.416952:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572876.416960:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572876.416965:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572876.416971:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572876.416978:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572876.416983:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572876.416991:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572876.416993:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572876.416996:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572877.440971:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572877.440978:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572877.440983:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572877.440989:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572877.440993:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572877.440998:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572877.441004:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572877.441009:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572877.441016:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572878.464929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572878.464935:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572878.464940:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572878.464944:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572878.464951:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572878.464956:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572878.464961:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572878.464965:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572878.464968:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572879.169098:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572879.169102:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572879.169106:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572879.488907:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572879.488912:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572879.488917:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572879.488922:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572879.488929:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572879.488933:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572879.488943:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572879.488945:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572879.488964:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572880.512962:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572880.512969:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572880.512974:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572880.512979:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572880.512984:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572880.512989:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572880.512998:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572880.513002:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572880.513006:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572880.768931:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308240621, at 4308240640 00000100:00000010:0.0:1788572880.768947:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea75a0. 00000100:00000001:0.0:1788572881.536908:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572881.536915:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572881.536921:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572881.536929:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572881.536934:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572881.536939:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572881.536950:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572881.536954:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572881.536956:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572882.048976:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572882.048985:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572882.048987:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572882.048992:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572882.048998:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572882.049012:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13569 00000400:00000200:0.0:1788572882.049024:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572882.049031:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572882.049036:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572882.049040:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049044:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572882.049048:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049050:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572882.049072:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572882.049075:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572882.049078:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572882.049082:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572882.049084:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572882.049087:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572882.049091:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049093:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572882.049096:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049098:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572882.049101:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572882.049105:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049108:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572882.049111:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572882.049115:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572882.049118:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.049120:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572882.049123:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572882.049129:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788572882.049133:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572882.049137:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572882.049139:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.049142:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572882.049145:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572882.049150:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572882.049158:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572882.049165:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572882.049175:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516502144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572882.049185:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572882.049189:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572882.049194:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516502144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572882.049478:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572882.049482:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572882.049486:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572882.049487:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572882.049489:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572882.049496:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572882.049501:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13569 00000400:00000200:0.0:1788572882.049507:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572882.049513:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572882.049518:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572882.049524:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049528:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572882.049534:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049541:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572882.049546:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572882.049549:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572882.049553:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572882.049559:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572882.049570:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572882.049574:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572882.049579:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049585:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572882.049590:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049595:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572882.049602:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572882.049609:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049613:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572882.049626:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572882.049636:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572882.049643:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.049648:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572882.049653:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572882.049659:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.049664:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572882.049670:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3100. 00000020:00000040:0.0:1788572882.049678:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572882.049685:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572882.049689:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.049694:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572882.049699:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572882.049703:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572882.049712:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572882.049719:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572882.049729:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3100 x1875440516502272/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572882.049745:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572882.049749:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572882.049755:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053a4b590] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3100 x1875440516502272/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572882.049772:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572882.049778:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572882.049786:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572882.049791:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572882.049794:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572882.049798:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572882.049806:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13569 00000400:00000200:0.0:1788572882.049834:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572882.049844:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572882.049848:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572882.049862:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049868:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572882.049875:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049879:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572882.049884:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572882.049886:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572882.049889:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572882.049893:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572882.049897:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572882.049904:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572882.049912:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049916:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572882.049921:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049926:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572882.049931:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572882.049939:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572882.049942:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572882.049947:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572882.049952:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572882.049958:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.049960:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572882.049963:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572882.049968:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.049972:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572882.049980:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788572882.049985:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572882.049992:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572882.049995:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.049999:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572882.050001:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572882.050006:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572882.050013:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788572882.050024:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572882.050034:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516502400/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572882.050050:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572882.050052:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572882.050058:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516502400/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572882.050072:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572882.050077:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572882.050083:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572882.050088:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572882.050091:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572882.050098:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572882.050105:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13569 00000400:00000200:0.0:1788572882.050115:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572882.050122:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572882.050125:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572882.050132:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.050137:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572882.050143:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.050148:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572882.050151:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572882.050153:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572882.050158:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572882.050163:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572882.050167:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572882.050172:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572882.050179:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.050185:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572882.050197:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.050203:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572882.050209:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572882.050217:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572882.050225:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572882.050229:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572882.050235:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572882.050240:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.050245:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572882.050250:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572882.050256:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.050261:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572882.050268:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788572882.050272:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572882.050279:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572882.050282:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.050287:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572882.050290:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572882.050294:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572882.050300:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788572882.050305:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572882.050316:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516502528/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572882.050331:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572882.050335:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572882.050343:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516502528/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572882.050359:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572882.050364:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572882.050374:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572882.050394:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572882.050404:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572882.050410:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572882.050417:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13569 00000400:00000200:0.0:1788572882.050428:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572882.050437:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572882.050441:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572882.050447:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.050453:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572882.050459:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.050464:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572882.050471:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572882.050474:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572882.050479:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572882.050485:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572882.050489:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572882.050494:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572882.050498:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.050501:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572882.050505:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572882.050508:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572882.050514:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572882.050516:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572882.050518:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572882.050519:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572882.050526:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572882.050530:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.050532:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572882.050534:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572882.050535:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.050537:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572882.050548:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788572882.050551:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572882.050556:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572882.050558:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572882.050562:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572882.050565:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572882.050567:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572882.050572:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572882.050576:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572882.050584:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516502656/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572882.050596:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572882.050600:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572882.050607:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516502656/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572882.050618:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572882.050625:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13579) 00000100:00000001:0.0:1788572883.584953:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572883.584964:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572883.584970:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572883.584976:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572883.584982:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572883.584987:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572883.584992:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572883.584996:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572883.584999:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572884.289002:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572884.289004:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572884.289007:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572884.608914:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572884.608922:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572884.608927:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572884.608933:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572884.608948:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572884.608951:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572884.608979:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572884.608980:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572884.608984:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572885.632908:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572885.632915:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572885.632917:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572885.632922:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572885.632927:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572885.632931:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572885.632937:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572885.632939:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572885.632942:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572885.889054:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308245742, at 4308245760 00000100:00000010:0.0:1788572885.889060:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f60. 00000100:00000001:0.0:1788572886.656922:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572886.656926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572886.656929:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572886.656936:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572886.656940:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572886.656944:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572886.656948:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572886.656950:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572886.656955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572887.168927:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572887.168939:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572887.168942:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572887.168946:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572887.168950:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572887.168956:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13574 00000400:00000200:0.0:1788572887.168965:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572887.168970:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572887.168973:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572887.168975:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.168978:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572887.168994:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.168996:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572887.168999:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572887.169000:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572887.169002:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572887.169004:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572887.169006:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572887.169008:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572887.169009:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169011:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572887.169012:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169013:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572887.169015:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572887.169017:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169018:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572887.169020:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572887.169022:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572887.169024:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169025:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572887.169026:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572887.169031:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788572887.169033:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572887.169036:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572887.169037:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169039:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572887.169042:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572887.169044:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572887.169050:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788572887.169054:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572887.169060:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516502784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572887.169068:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572887.169073:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572887.169077:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516502784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572887.169092:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572887.169094:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572887.169096:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572887.169097:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572887.169098:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572887.169100:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572887.169102:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13574 00000400:00000200:0.0:1788572887.169105:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572887.169107:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572887.169108:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572887.169110:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169112:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572887.169113:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169115:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572887.169117:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572887.169119:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572887.169121:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572887.169123:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572887.169126:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572887.169128:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572887.169132:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169134:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572887.169137:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169139:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572887.169143:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572887.169148:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169155:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572887.169156:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572887.169160:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572887.169163:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169165:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572887.169170:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572887.169173:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169175:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572887.169178:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788572887.169180:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572887.169184:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572887.169186:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169188:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572887.169191:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572887.169193:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572887.169196:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788572887.169201:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572887.169206:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516502912/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572887.169217:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572887.169220:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572887.169223:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516502912/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572887.169234:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572887.169236:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572887.169241:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572887.169243:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572887.169244:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572887.169246:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572887.169248:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13574 00000400:00000200:0.0:1788572887.169251:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572887.169262:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572887.169264:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572887.169267:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169269:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572887.169271:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169272:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572887.169273:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572887.169274:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572887.169274:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572887.169276:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572887.169276:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572887.169280:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572887.169282:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169284:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572887.169285:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169292:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572887.169296:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572887.169302:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169306:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572887.169310:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572887.169314:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572887.169319:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169323:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572887.169326:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572887.169331:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169335:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572887.169339:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788572887.169342:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572887.169345:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572887.169346:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169350:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572887.169355:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572887.169358:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572887.169364:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788572887.169366:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572887.169372:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516503040/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572887.169381:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572887.169384:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572887.169390:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516503040/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572887.169399:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572887.169402:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572887.169406:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572887.169408:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572887.169410:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572887.169414:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572887.169418:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13574 00000400:00000200:0.0:1788572887.169423:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572887.169457:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572887.169461:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572887.169466:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169469:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572887.169472:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169474:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572887.169475:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572887.169476:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572887.169477:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572887.169478:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572887.169480:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572887.169483:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572887.169491:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169493:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572887.169497:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169499:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572887.169503:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572887.169506:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169510:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572887.169513:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572887.169516:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572887.169519:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169522:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572887.169525:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572887.169527:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169530:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572887.169534:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788572887.169537:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572887.169539:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572887.169542:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169544:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572887.169546:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572887.169550:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572887.169552:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788572887.169556:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572887.169563:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516503168/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572887.169574:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572887.169576:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572887.169582:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516503168/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572887.169587:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572887.169589:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572887.169597:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572887.169599:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572887.169601:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572887.169604:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572887.169608:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13574 00000400:00000200:0.0:1788572887.169615:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572887.169619:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572887.169622:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572887.169625:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169628:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572887.169631:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169633:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572887.169635:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572887.169637:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572887.169639:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572887.169641:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572887.169644:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572887.169648:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572887.169653:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169657:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572887.169659:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169665:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572887.169668:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572887.169670:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572887.169671:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572887.169673:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572887.169676:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572887.169678:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169680:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572887.169682:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572887.169688:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169691:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572887.169695:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788572887.169697:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572887.169701:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572887.169702:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.169703:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572887.169704:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572887.169705:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572887.169708:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572887.169712:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572887.169716:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516503296/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572887.169723:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572887.169725:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572887.169730:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516503296/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572887.169737:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572887.169742:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13584) 00000100:00000001:0.0:1788572887.680914:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572887.680922:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572887.680926:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572887.680933:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572887.680938:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572887.680944:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572887.680951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572887.680954:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572887.680957:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572887.680986:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572887.680990:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572887.680992:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572887.680994:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572887.680996:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572887.680998:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572887.681006:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572887.681008:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572887.681010:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572888.704921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572888.704927:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572888.704930:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572888.704934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572888.704941:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572888.704946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572888.704949:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572888.704953:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572888.704956:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572889.409007:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572889.409010:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572889.409013:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572889.728910:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572889.728917:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572889.728920:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572889.728927:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572889.728934:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572889.728940:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572889.728958:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572889.728960:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572889.728962:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572890.752906:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572890.752911:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572890.752914:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572890.752921:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572890.752925:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572890.752929:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572890.752934:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572890.752938:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572890.752943:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572891.008901:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308250861, at 4308250880 00000100:00000010:0.0:1788572891.008916:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea77e0. 00000100:00000001:0.0:1788572891.776905:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572891.776913:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572891.776929:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572891.776934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572891.776940:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572891.776943:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572891.776954:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572891.776957:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572891.776960:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572892.288946:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572892.288957:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572892.288959:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572892.288964:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572892.288971:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572892.288979:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13579 00000400:00000200:0.0:1788572892.288989:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572892.288997:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572892.289000:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572892.289004:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289007:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572892.289009:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289011:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572892.289015:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572892.289017:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572892.289020:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572892.289023:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572892.289025:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572892.289028:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572892.289031:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289033:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572892.289035:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289037:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572892.289040:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572892.289043:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289063:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572892.289066:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572892.289069:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572892.289072:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289074:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572892.289076:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572892.289083:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788572892.289087:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572892.289090:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572892.289092:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289095:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572892.289097:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572892.289101:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572892.289108:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788572892.289115:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572892.289124:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516503424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572892.289134:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572892.289138:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572892.289143:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516503424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572892.289162:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572892.289165:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572892.289168:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572892.289169:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572892.289170:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572892.289172:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572892.289174:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13579 00000400:00000200:0.0:1788572892.289177:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572892.289180:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572892.289181:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572892.289184:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289193:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572892.289197:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289200:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572892.289202:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572892.289203:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572892.289204:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572892.289206:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572892.289210:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572892.289213:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572892.289219:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289222:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572892.289226:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289231:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572892.289236:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572892.289239:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289242:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572892.289244:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572892.289250:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572892.289253:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289256:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572892.289260:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572892.289264:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289266:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572892.289268:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788572892.289272:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572892.289274:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572892.289276:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289279:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572892.289282:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572892.289283:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572892.289287:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788572892.289293:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572892.289302:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516503552/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572892.289309:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572892.289312:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572892.289316:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516503552/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572892.289324:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572892.289326:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572892.289328:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572892.289331:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572892.289334:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572892.289338:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572892.289342:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13579 00000400:00000200:0.0:1788572892.289347:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572892.289350:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572892.289354:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572892.289360:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289364:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572892.289368:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289371:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572892.289375:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572892.289377:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572892.289379:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572892.289381:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572892.289386:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572892.289388:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572892.289394:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289396:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572892.289399:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289405:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572892.289408:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572892.289411:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289414:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572892.289417:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572892.289420:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572892.289422:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289424:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572892.289426:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572892.289427:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289428:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572892.289431:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788572892.289434:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572892.289437:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572892.289438:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289440:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572892.289442:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572892.289444:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572892.289447:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572892.289450:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572892.289455:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516503680/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572892.289465:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572892.289468:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572892.289473:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516503680/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572892.289486:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572892.289504:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572892.289507:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572892.289510:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572892.289512:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572892.289522:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572892.289527:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13579 00000400:00000200:0.0:1788572892.289534:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572892.289538:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572892.289540:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572892.289545:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289546:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572892.289550:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289553:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572892.289557:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572892.289559:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572892.289561:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572892.289564:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572892.289568:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572892.289573:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572892.289589:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289594:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572892.289600:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289605:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572892.289610:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572892.289615:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289620:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572892.289621:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572892.289624:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572892.289628:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289631:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572892.289634:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572892.289636:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289640:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572892.289642:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788572892.289650:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572892.289655:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572892.289657:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289660:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572892.289662:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572892.289664:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572892.289667:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572892.289671:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572892.289684:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516503808/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572892.289692:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572892.289695:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572892.289700:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516503808/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572892.289712:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572892.289714:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572892.289718:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572892.289719:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572892.289721:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572892.289723:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572892.289725:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13579 00000400:00000200:0.0:1788572892.289731:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572892.289734:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572892.289739:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572892.289745:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289748:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572892.289752:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289755:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572892.289758:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572892.289760:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572892.289769:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572892.289772:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572892.289775:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572892.289777:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572892.289782:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289785:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572892.289789:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289793:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572892.289795:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572892.289801:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572892.289805:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572892.289826:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572892.289828:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572892.289833:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289836:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572892.289838:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572892.289844:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289848:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572892.289852:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788572892.289856:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572892.289859:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572892.289861:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572892.289863:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572892.289866:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572892.289868:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572892.289875:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572892.289879:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572892.289887:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516503936/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572892.289897:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572892.289898:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572892.289903:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516503936/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572892.289925:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572892.289930:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13589) 00000100:00000001:0.0:1788572892.800891:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572892.800896:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572892.800898:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572892.800901:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572892.800905:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572892.800907:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572892.800910:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572892.800912:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572892.800914:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572893.824970:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572893.824978:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572893.824984:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572893.824991:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572893.825004:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572893.825017:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572893.825025:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572893.825028:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572893.825030:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572894.529012:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572894.529015:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572894.529017:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572894.848918:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572894.848925:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572894.848929:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572894.848936:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572894.848944:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572894.848949:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572894.848957:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572894.848961:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572894.848969:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572895.872904:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572895.872911:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572895.872915:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572895.872921:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572895.872938:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572895.872943:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572895.872955:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572895.872957:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572895.872961:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572896.128914:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308255981, at 4308256000 00000100:00000010:0.0:1788572896.128929:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea79c0. 00000100:00000001:0.0:1788572896.896910:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572896.896918:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572896.896921:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572896.896930:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572896.896936:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572896.896941:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572896.896948:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572896.896952:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572896.896957:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572897.920896:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572897.920903:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572897.920904:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572897.920909:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572897.920913:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572897.920918:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572897.920924:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572897.920930:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572897.920938:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572897.920984:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572897.920991:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572897.920999:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572897.921004:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572897.921012:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572897.921015:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572897.921017:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572897.921019:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572897.921020:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572898.944940:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572898.944946:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572898.944950:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572898.944971:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572898.944974:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572898.944977:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572898.944980:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572898.944982:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572898.944984:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572899.649014:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572899.649018:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572899.649022:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572899.968895:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572899.968903:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572899.968907:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572899.968915:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572899.968922:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572899.968929:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572899.968943:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572899.968948:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572899.968951:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572900.992978:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572900.992983:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572900.992985:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572900.992989:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572900.992999:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572900.993005:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572900.993021:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572900.993026:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572900.993030:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572900.993047:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572900.993050:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572900.993052:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572900.993054:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572900.993058:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572900.993062:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572900.993064:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572900.993065:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572900.993067:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572901.248938:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308261102, at 4308261120 00000100:00000010:0.0:1788572901.248956:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea75a0. 00000100:00000001:0.0:1788572902.016920:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572902.016925:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572902.016927:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572902.016933:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572902.016940:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572902.016943:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572902.016960:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572902.016965:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572902.016968:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572902.529014:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572902.529029:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572902.529035:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572902.529042:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572902.529050:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572902.529061:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13589 00000400:00000200:0.0:1788572902.529072:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572902.529080:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572902.529084:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572902.529088:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529092:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572902.529095:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529098:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572902.529102:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572902.529104:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572902.529107:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572902.529110:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572902.529112:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572902.529114:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572902.529117:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529118:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572902.529120:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529121:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572902.529146:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572902.529150:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529152:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572902.529156:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572902.529159:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572902.529161:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.529163:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572902.529165:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572902.529171:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788572902.529175:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572902.529179:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572902.529180:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.529183:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572902.529186:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572902.529191:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572902.529198:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788572902.529206:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572902.529214:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516504704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572902.529224:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572902.529228:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572902.529233:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516504704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572902.529252:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572902.529254:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572902.529257:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572902.529258:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572902.529260:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572902.529262:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572902.529264:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13589 00000400:00000200:0.0:1788572902.529268:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572902.529276:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572902.529278:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572902.529282:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529289:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572902.529296:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529304:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572902.529306:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572902.529307:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572902.529309:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572902.529311:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572902.529316:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572902.529320:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572902.529332:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529337:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572902.529342:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529349:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572902.529356:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572902.529364:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529369:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572902.529377:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572902.529384:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572902.529388:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.529392:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572902.529396:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572902.529402:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.529405:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572902.529413:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788572902.529418:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572902.529427:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572902.529429:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.529433:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572902.529440:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572902.529446:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572902.529456:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788572902.529462:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572902.529470:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516504832/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572902.529487:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572902.529490:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572902.529494:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516504832/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572902.529510:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572902.529515:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572902.529521:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572902.529524:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572902.529532:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572902.529542:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572902.529552:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13589 00000400:00000200:0.0:1788572902.529571:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572902.529580:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572902.529586:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572902.529594:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529618:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572902.529623:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529626:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572902.529629:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572902.529630:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572902.529635:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572902.529637:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572902.529644:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572902.529652:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572902.529674:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529682:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572902.529689:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529695:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572902.529705:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572902.529720:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572902.529731:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572902.529738:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572902.529747:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572902.529756:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.529764:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572902.529768:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572902.529777:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.529779:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572902.529790:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788572902.529799:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572902.529806:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572902.529834:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.529837:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572902.529839:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572902.529842:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572902.529849:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788572902.529854:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572902.529864:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516504960/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572902.529889:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572902.529893:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572902.529905:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516504960/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572902.529922:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572902.529929:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572902.529942:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572902.529946:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572902.529949:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572902.529960:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572902.529968:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13589 00000400:00000200:0.0:1788572902.529984:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572902.529990:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572902.529997:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572902.530008:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.530019:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572902.530029:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.530043:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572902.530050:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572902.530054:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572902.530062:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572902.530070:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572902.530084:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572902.530094:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572902.530104:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.530112:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572902.530120:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.530126:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572902.530134:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572902.530142:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572902.530148:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572902.530154:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572902.530161:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572902.530170:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.530181:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572902.530187:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572902.530201:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.530206:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572902.530210:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788572902.530212:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572902.530216:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572902.530220:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.530224:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572902.530225:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572902.530230:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572902.530234:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788572902.530240:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572902.530250:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516505088/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572902.530266:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572902.530271:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572902.530276:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516505088/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572902.530294:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572902.530304:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572902.530310:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572902.530315:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572902.530317:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572902.530325:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572902.530331:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13589 00000400:00000200:0.0:1788572902.530339:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572902.530348:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572902.530354:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572902.530358:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.530360:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572902.530365:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.530376:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572902.530384:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572902.530387:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572902.530390:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572902.530392:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572902.530399:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572902.530407:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572902.530412:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.530415:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572902.530419:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572902.530422:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572902.530428:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572902.530438:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572902.530449:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572902.530458:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572902.530466:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572902.530474:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.530483:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572902.530490:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572902.530502:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.530510:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572902.530523:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788572902.530537:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572902.530546:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572902.530552:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572902.530559:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572902.530565:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572902.530570:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572902.530585:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572902.530594:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572902.530620:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516505216/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572902.530674:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572902.530682:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572902.530696:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516505216/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572902.530715:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572902.530726:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13600) 00000100:00000001:0.0:1788572903.040905:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572903.040917:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572903.040922:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572903.040931:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572903.040937:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572903.040944:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572903.040962:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572903.040966:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572903.040969:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572904.064959:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572904.064971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572904.064976:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572904.064986:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572904.064995:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572904.065001:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572904.065026:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572904.065031:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572904.065035:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572904.769022:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572904.769026:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572904.769029:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572905.088929:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572905.088938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572905.088941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572905.088947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572905.088953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572905.088960:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572905.088970:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572905.089001:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572905.089010:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572906.112946:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572906.112955:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572906.112962:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572906.112968:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572906.112978:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572906.112993:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572906.113018:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572906.113021:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572906.113024:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572906.368924:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308266222, at 4308266240 00000100:00000010:0.0:1788572906.368943:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7180. 00000100:00080000:0.0:1788572907.648899:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572907.648908:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572907.648911:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572907.648914:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572907.648919:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572907.648924:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13595 00000400:00000200:0.0:1788572907.648933:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572907.648939:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572907.648941:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572907.648944:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.648947:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572907.648949:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.648951:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572907.648954:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572907.648956:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572907.648958:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572907.648960:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572907.648962:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572907.648964:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572907.648967:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.648983:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572907.648986:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.648988:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572907.648991:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572907.648995:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572907.648997:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572907.649000:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572907.649002:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572907.649005:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649007:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572907.649008:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572907.649013:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788572907.649016:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572907.649020:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572907.649021:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649023:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572907.649025:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572907.649030:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572907.649053:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788572907.649060:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572907.649070:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516505344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572907.649080:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572907.649085:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572907.649090:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516505344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572907.649117:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572907.649121:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572907.649125:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572907.649127:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572907.649129:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572907.649132:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572907.649151:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13595 00000400:00000200:0.0:1788572907.649161:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572907.649167:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572907.649170:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572907.649174:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649179:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572907.649184:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649189:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572907.649195:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572907.649197:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572907.649202:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572907.649206:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572907.649210:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572907.649215:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572907.649224:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649230:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572907.649235:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649241:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572907.649246:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572907.649251:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649255:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572907.649259:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572907.649269:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572907.649275:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649280:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572907.649285:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572907.649292:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649298:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572907.649304:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788572907.649308:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572907.649313:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572907.649322:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649325:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572907.649329:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572907.649333:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572907.649341:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572907.649349:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572907.649361:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516505472/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572907.649377:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572907.649383:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572907.649391:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516505472/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572907.649410:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572907.649414:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572907.649422:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572907.649425:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572907.649428:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572907.649436:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572907.649442:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13595 00000400:00000200:0.0:1788572907.649452:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572907.649459:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572907.649464:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572907.649473:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649476:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572907.649479:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649482:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572907.649487:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572907.649490:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572907.649494:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572907.649498:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572907.649503:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572907.649515:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572907.649525:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649530:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572907.649533:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649539:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572907.649546:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572907.649550:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649554:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572907.649559:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572907.649563:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572907.649567:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649572:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572907.649575:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572907.649582:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649587:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572907.649593:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788572907.649599:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572907.649606:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572907.649610:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649614:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572907.649618:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572907.649622:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572907.649628:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572907.649634:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572907.649645:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516505600/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572907.649677:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572907.649681:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572907.649688:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516505600/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572907.649701:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572907.649713:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572907.649718:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572907.649720:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572907.649721:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572907.649725:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572907.649729:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13595 00000400:00000200:0.0:1788572907.649733:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572907.649737:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572907.649739:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572907.649742:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649745:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572907.649747:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649749:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572907.649752:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572907.649753:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572907.649755:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572907.649757:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572907.649759:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572907.649761:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572907.649764:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649766:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572907.649767:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649769:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572907.649773:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572907.649775:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649778:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572907.649779:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572907.649783:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572907.649785:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649788:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572907.649797:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572907.649799:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649801:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572907.649804:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788572907.649836:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572907.649840:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572907.649842:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649843:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572907.649845:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572907.649847:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572907.649852:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572907.649856:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572907.649863:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516505728/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572907.649871:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572907.649873:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572907.649877:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516505728/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572907.649885:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572907.649888:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572907.649891:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572907.649893:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572907.649894:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572907.649897:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572907.649899:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13595 00000400:00000200:0.0:1788572907.649905:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572907.649908:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572907.649911:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572907.649913:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649916:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572907.649918:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649926:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572907.649928:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572907.649930:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572907.649932:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572907.649934:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572907.649936:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572907.649938:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572907.649943:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649945:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572907.649947:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649949:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572907.649951:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572907.649954:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572907.649956:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572907.649958:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572907.649962:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572907.649964:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649966:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572907.649968:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572907.649971:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649972:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572907.649976:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788572907.649978:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572907.649980:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572907.649982:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572907.649983:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572907.649985:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572907.649987:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572907.649990:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788572907.649994:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572907.649998:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516505856/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572907.650017:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572907.650019:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572907.650023:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516505856/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572907.650030:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572907.650036:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13605) 00000100:00000001:0.0:1788572909.185009:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572909.185012:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572909.185014:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572909.185018:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572909.185021:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572909.185024:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572909.185027:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572909.185029:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572909.185031:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572909.888988:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572909.888991:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572909.888994:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572910.208921:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572910.208925:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572910.208930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572910.208934:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572910.208939:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572910.208943:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572910.208948:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572910.208951:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572910.208953:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572911.232954:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572911.232961:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572911.232965:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572911.232972:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572911.232978:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572911.232986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572911.232996:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572911.233013:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572911.233016:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572911.488896:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308271342, at 4308271360 00000100:00000010:0.0:1788572911.488911:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7360. 00000100:00000001:0.0:1788572912.256963:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572912.256968:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572912.256974:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572912.256979:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572912.256983:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572912.256986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572912.256990:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572912.256992:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572912.256994:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572912.768901:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572912.768910:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572912.768913:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572912.768917:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572912.768921:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572912.768928:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13600 00000400:00000200:0.0:1788572912.768938:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572912.768944:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572912.768946:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572912.768950:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.768952:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572912.768954:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.768956:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572912.768960:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572912.768961:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572912.768963:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572912.768965:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572912.768967:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572912.768969:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572912.768971:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.768989:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572912.768991:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.768992:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572912.768995:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572912.768997:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572912.768999:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572912.769001:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572912.769003:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572912.769005:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769007:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572912.769009:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572912.769013:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788572912.769016:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572912.769019:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572912.769021:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769023:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572912.769025:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572912.769029:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572912.769035:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788572912.769041:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572912.769048:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516505984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572912.769056:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572912.769059:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572912.769063:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516505984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572912.769080:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572912.769082:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572912.769085:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572912.769086:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572912.769088:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572912.769090:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572912.769097:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13600 00000400:00000200:0.0:1788572912.769101:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572912.769119:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572912.769124:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572912.769130:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769134:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572912.769139:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769141:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572912.769145:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572912.769148:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572912.769151:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572912.769154:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572912.769159:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572912.769162:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572912.769167:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769170:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572912.769174:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769176:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572912.769178:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572912.769184:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769187:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572912.769188:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572912.769194:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572912.769198:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769200:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572912.769204:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572912.769207:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769209:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572912.769213:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788572912.769216:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572912.769218:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572912.769227:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769230:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572912.769232:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572912.769236:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572912.769242:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788572912.769247:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572912.769255:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516506112/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572912.769266:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572912.769271:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572912.769275:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516506112/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572912.769286:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572912.769291:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572912.769296:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572912.769299:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572912.769302:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572912.769306:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572912.769310:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13600 00000400:00000200:0.0:1788572912.769319:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572912.769325:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572912.769330:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572912.769333:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769335:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572912.769340:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769344:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572912.769348:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572912.769351:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572912.769353:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572912.769357:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572912.769361:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572912.769369:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572912.769374:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769379:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572912.769385:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769389:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572912.769397:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572912.769405:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769410:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572912.769415:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572912.769420:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572912.769425:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769430:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572912.769433:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572912.769439:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769442:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572912.769447:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788572912.769453:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572912.769459:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572912.769462:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769464:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572912.769465:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572912.769466:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572912.769470:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572912.769475:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572912.769485:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516506240/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572912.769494:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572912.769498:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572912.769507:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516506240/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572912.769520:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572912.769529:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572912.769535:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572912.769537:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572912.769539:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572912.769545:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572912.769549:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13600 00000400:00000200:0.0:1788572912.769555:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572912.769560:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572912.769563:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572912.769569:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769574:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572912.769579:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769584:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572912.769587:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572912.769590:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572912.769592:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572912.769595:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572912.769599:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572912.769604:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572912.769609:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769613:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572912.769616:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769621:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572912.769626:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572912.769632:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769636:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572912.769638:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572912.769642:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572912.769646:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769649:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572912.769656:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572912.769661:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769663:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572912.769666:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788572912.769677:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572912.769683:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572912.769686:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769688:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572912.769691:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572912.769694:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572912.769698:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572912.769705:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572912.769727:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516506368/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572912.769740:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572912.769745:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572912.769754:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516506368/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572912.769767:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572912.769772:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572912.769775:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572912.769776:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572912.769777:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572912.769779:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572912.769785:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13600 00000400:00000200:0.0:1788572912.769794:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572912.769803:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572912.769838:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572912.769843:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769847:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572912.769852:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769864:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572912.769868:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572912.769871:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572912.769875:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572912.769879:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572912.769881:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572912.769883:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572912.769887:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769890:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572912.769895:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769899:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572912.769902:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572912.769908:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572912.769912:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572912.769916:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572912.769921:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572912.769927:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769930:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572912.769934:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572912.769940:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769943:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572912.769949:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788572912.769952:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572912.769957:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572912.769961:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572912.769964:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572912.769966:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572912.769969:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572912.769975:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572912.769981:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572912.769989:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516506496/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572912.770010:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572912.770015:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572912.770022:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516506496/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572912.770039:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572912.770045:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13610) 00000100:00000001:0.0:1788572913.280945:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572913.280952:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572913.280957:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572913.280966:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572913.280973:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572913.280979:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572913.280994:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572913.280997:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572913.281000:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572914.304902:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572914.304910:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572914.304913:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572914.304920:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572914.304924:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572914.304929:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572914.304946:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572914.304950:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572914.304954:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572914.305116:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572914.305118:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572914.305119:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572914.305121:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572914.305122:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572914.305124:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572914.305125:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572914.305126:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572914.305127:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572915.008999:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572915.009005:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572915.009033:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572915.328930:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572915.328938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572915.328946:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572915.328953:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572915.328961:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572915.328966:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572915.328979:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572915.328992:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572915.328995:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572915.329010:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572915.329015:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572915.329017:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572915.329020:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572915.329022:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572915.329024:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572915.329026:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572915.329028:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572915.329030:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572916.352966:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572916.352979:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572916.352983:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572916.352992:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572916.353002:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572916.353006:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572916.353010:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572916.353015:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572916.353019:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572916.608923:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308276461, at 4308276480 00000100:00000010:0.0:1788572916.608937:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7600. 00000100:00000001:0.0:1788572917.376950:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572917.376967:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572917.376973:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572917.376979:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572917.376988:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572917.376995:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572917.377007:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572917.377031:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572917.377039:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572917.888916:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572917.888929:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572917.888932:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572917.888936:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572917.888941:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572917.888949:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13605 00000400:00000200:0.0:1788572917.888960:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572917.888967:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572917.888970:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572917.888973:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.888977:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572917.888980:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.888983:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572917.888987:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572917.888989:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572917.888992:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572917.888995:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572917.888998:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572917.889001:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572917.889004:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889006:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572917.889008:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889010:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572917.889014:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572917.889018:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889020:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572917.889024:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572917.889027:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572917.889030:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.889051:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572917.889054:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572917.889061:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3100. 00000020:00000040:0.0:1788572917.889065:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572917.889070:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572917.889072:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.889075:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572917.889078:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572917.889082:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572917.889091:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788572917.889098:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572917.889109:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3100 x1875440516506624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572917.889121:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572917.889126:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572917.889131:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053a4b590] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3100 x1875440516506624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572917.889153:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572917.889158:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572917.889162:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572917.889163:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572917.889166:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572917.889170:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572917.889173:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13605 00000400:00000200:0.0:1788572917.889179:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572917.889187:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572917.889192:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572917.889200:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889206:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572917.889210:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889217:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572917.889228:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572917.889232:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572917.889237:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572917.889243:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572917.889247:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572917.889254:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572917.889262:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889266:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572917.889270:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889274:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572917.889278:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572917.889282:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889287:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572917.889290:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572917.889296:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572917.889301:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.889307:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572917.889312:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572917.889317:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.889320:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572917.889329:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788572917.889335:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572917.889342:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572917.889344:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.889347:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572917.889348:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572917.889351:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572917.889358:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788572917.889367:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572917.889379:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516506752/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572917.889395:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572917.889406:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572917.889413:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516506752/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572917.889432:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572917.889438:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572917.889445:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572917.889449:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572917.889453:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572917.889458:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572917.889464:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13605 00000400:00000200:0.0:1788572917.889474:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572917.889478:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572917.889481:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572917.889487:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889492:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572917.889496:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889500:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572917.889502:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572917.889505:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572917.889507:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572917.889511:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572917.889513:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572917.889515:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572917.889521:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889522:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572917.889524:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889525:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572917.889526:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572917.889528:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889537:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572917.889540:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572917.889543:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572917.889548:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.889551:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572917.889554:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572917.889557:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.889561:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572917.889566:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788572917.889573:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572917.889578:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572917.889581:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.889585:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572917.889589:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572917.889594:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572917.889602:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788572917.889609:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572917.889619:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516506880/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572917.889649:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572917.889653:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572917.889662:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516506880/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572917.889675:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572917.889679:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572917.889682:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572917.889684:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572917.889685:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572917.889690:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572917.889695:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13605 00000400:00000200:0.0:1788572917.889703:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572917.889714:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572917.889717:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572917.889720:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889724:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572917.889739:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889744:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572917.889747:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572917.889751:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572917.889753:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572917.889756:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572917.889762:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572917.889767:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572917.889783:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889788:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572917.889794:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889800:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572917.889806:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572917.889841:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572917.889845:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572917.889849:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572917.889855:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572917.889860:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.889865:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572917.889870:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572917.889876:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.889879:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572917.889886:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788572917.889891:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572917.889896:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572917.889899:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.889903:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572917.889912:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572917.889916:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572917.889924:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572917.889930:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572917.889938:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516507008/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572917.889961:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572917.889965:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572917.889972:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516507008/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572917.889993:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572917.889998:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572917.890005:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572917.890007:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572917.890009:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572917.890012:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572917.890016:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13605 00000400:00000200:0.0:1788572917.890022:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572917.890027:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572917.890030:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572917.890036:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.890040:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572917.890045:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.890048:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572917.890051:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572917.890052:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572917.890055:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572917.890057:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572917.890062:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572917.890066:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572917.890074:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.890078:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572917.890082:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572917.890086:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572917.890090:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572917.890095:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572917.890100:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572917.890104:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572917.890107:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572917.890113:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.890117:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572917.890118:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572917.890122:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.890126:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572917.890131:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788572917.890135:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572917.890139:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572917.890142:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572917.890147:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572917.890149:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572917.890153:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572917.890156:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788572917.890161:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572917.890168:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516507136/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572917.890178:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572917.890181:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572917.890184:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516507136/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572917.890190:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572917.890196:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13615) 00000100:00000001:0.0:1788572918.400929:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572918.400935:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572918.400937:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572918.400941:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572918.400945:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572918.400949:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572918.400969:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572918.400973:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572918.400982:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572919.424949:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572919.424959:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572919.424964:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572919.424973:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572919.424980:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572919.424987:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572919.425004:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572919.425008:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572919.425012:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572920.129026:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572920.129030:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572920.129034:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572920.448896:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572920.448902:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572920.448907:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572920.448916:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572920.448923:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572920.448927:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572920.448940:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572920.448944:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572920.448948:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788572920.961100:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572920.961117:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572920.961124:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572920.961127:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572921.472972:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572921.472979:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572921.472983:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572921.472989:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572921.473004:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572921.473009:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572921.473015:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572921.473018:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572921.473022:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572921.728948:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308281581, at 4308281600 00000100:00000010:0.0:1788572921.728966:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ea0. 00000100:00000001:0.0:1788572922.496930:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572922.496936:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572922.496940:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572922.496946:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572922.496953:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572922.496957:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572922.496960:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572922.496964:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572922.496968:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572923.520940:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572923.520950:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572923.520956:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572923.520965:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572923.520972:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572923.520978:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572923.521004:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572923.521008:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572923.521012:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572924.544928:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572924.544937:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572924.544941:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572924.544947:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572924.544954:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572924.544959:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572924.544966:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572924.544971:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572924.544978:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572925.248979:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572925.248982:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572925.248985:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572925.568931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572925.568941:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572925.568946:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572925.568955:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572925.568963:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572925.568968:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572925.568994:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572925.568999:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572925.569004:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572926.592957:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572926.592966:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572926.592971:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572926.592980:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572926.592985:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572926.592990:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572926.592997:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572926.593001:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572926.593008:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572926.593038:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572926.593041:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572926.593043:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572926.593045:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572926.593047:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572926.593050:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572926.593051:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572926.593053:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572926.593055:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572926.848898:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308286701, at 4308286720 00000100:00000010:0.0:1788572926.848918:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ba0. 00000100:00000001:0.0:1788572928.641003:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572928.641010:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572928.641014:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572928.641019:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572928.641027:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572928.641034:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572928.641039:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572928.641043:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572928.641067:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572929.665091:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572929.665102:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572929.665106:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572929.665115:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572929.665126:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572929.665131:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572929.665140:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572929.665146:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572929.665153:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572930.369051:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572930.369059:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572930.369065:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572930.689005:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572930.689013:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572930.689019:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572930.689027:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572930.689035:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572930.689047:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572930.689059:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572930.689064:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572930.689069:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572931.713006:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572931.713014:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572931.713020:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572931.713037:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572931.713045:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572931.713049:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572931.713080:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572931.713086:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572931.713092:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572931.968966:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308291821, at 4308291840 00000100:00000010:0.0:1788572931.968987:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c00. 00000100:00000001:0.0:1788572932.736980:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572932.736992:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572932.736996:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572932.737002:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572932.737023:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572932.737037:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572932.737067:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572932.737073:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572932.737079:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572932.737130:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572932.737137:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572932.737143:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572932.737149:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572932.737171:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572932.737174:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572932.737178:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572932.737181:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572932.737184:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572933.760962:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572933.760969:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572933.760974:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572933.760980:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572933.760986:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572933.760992:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572933.761012:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572933.761014:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572933.761017:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572933.761032:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572933.761034:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572933.761036:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572933.761038:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572933.761041:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572933.761043:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572933.761046:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572933.761048:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572933.761050:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572934.784933:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572934.784940:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572934.784945:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572934.784953:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572934.784974:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572934.784980:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572934.785002:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572934.785005:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572934.785008:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572935.489010:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572935.489013:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572935.489015:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572935.808923:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572935.808933:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572935.808938:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572935.808947:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572935.808954:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572935.808981:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572935.809004:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572935.809009:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572935.809013:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572936.833075:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572936.833079:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572936.833081:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572936.833084:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572936.833086:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572936.833088:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572936.833753:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572936.833756:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572936.833758:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572937.088926:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308296941, at 4308296960 00000100:00000010:0.0:1788572937.088941:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7d80. 00000100:00000001:0.0:1788572937.857007:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572937.857016:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572937.857023:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572937.857038:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572937.857050:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572937.857061:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572937.857086:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572937.857090:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572937.857095:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572938.881026:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572938.881034:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572938.881051:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572938.881057:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572938.881062:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572938.881067:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572938.881073:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572938.881076:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572938.881081:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572939.904922:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572939.904930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572939.904936:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572939.904956:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572939.904965:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572939.904970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572939.904974:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572939.904976:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572939.904978:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572940.608998:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572940.609002:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572940.609004:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572940.928920:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572940.928930:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572940.928935:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572940.928944:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572940.928954:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572940.928960:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572940.928987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572940.928996:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572940.929001:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572940.929037:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572940.929041:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572940.929047:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572940.929051:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572940.929058:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572940.929062:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572940.929064:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572940.929065:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572940.929067:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572941.952960:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572941.952975:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572941.952980:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572941.952986:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572941.952993:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572941.952998:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572941.953003:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572941.953008:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572941.953014:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572942.208940:0:259895:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308302061, at 4308302080 00000100:00000010:0.0:1788572942.208965:0:259895:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7840. 00000100:00000001:0.0:1788572942.976940:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572942.976946:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572942.976950:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572942.976958:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572942.976967:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572942.976975:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572942.976990:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572942.976995:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572942.977000:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572943.488914:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572943.488923:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572943.488927:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572943.488931:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572943.488936:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572943.488943:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13630 00000400:00000200:0.0:1788572943.488953:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572943.488960:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572943.488963:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572943.488967:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.488970:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572943.488973:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.488975:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572943.488978:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572943.488996:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572943.488999:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572943.489001:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572943.489003:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572943.489005:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572943.489007:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489010:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572943.489011:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489012:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572943.489014:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572943.489016:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489032:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572943.489034:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572943.489037:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572943.489040:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489041:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572943.489043:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572943.489048:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788572943.489051:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572943.489054:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572943.489056:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489058:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572943.489060:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572943.489064:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572943.489070:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788572943.489076:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572943.489083:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516509824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572943.489092:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572943.489095:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572943.489099:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516509824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572943.489127:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572943.489129:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572943.489132:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572943.489133:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572943.489134:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572943.489136:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572943.489138:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13630 00000400:00000200:0.0:1788572943.489141:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572943.489143:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572943.489145:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572943.489148:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489150:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572943.489152:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489154:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572943.489156:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572943.489157:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572943.489158:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572943.489159:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572943.489162:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572943.489164:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572943.489168:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489172:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572943.489176:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489178:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572943.489182:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572943.489186:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489191:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572943.489199:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572943.489212:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572943.489219:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489224:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572943.489231:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572943.489234:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489236:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572943.489242:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788572943.489245:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572943.489250:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572943.489252:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489254:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572943.489257:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572943.489260:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572943.489264:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572943.489269:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572943.489276:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516509952/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572943.489285:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572943.489289:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572943.489298:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516509952/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572943.489330:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572943.489335:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572943.489340:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572943.489343:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572943.489345:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572943.489351:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572943.489356:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13630 00000400:00000200:0.0:1788572943.489364:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572943.489371:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572943.489375:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572943.489390:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489397:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572943.489405:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489412:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572943.489415:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572943.489417:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572943.489419:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572943.489423:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572943.489426:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572943.489431:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572943.489437:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489440:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572943.489442:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489447:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572943.489449:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572943.489454:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489458:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572943.489462:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572943.489466:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572943.489469:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489471:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572943.489472:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572943.489476:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489478:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572943.489492:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788572943.489500:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572943.489505:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572943.489508:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489511:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572943.489512:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572943.489514:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572943.489517:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788572943.489528:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572943.489537:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516510080/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572943.489545:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572943.489547:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572943.489559:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516510080/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572943.489573:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572943.489576:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572943.489581:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572943.489583:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572943.489585:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572943.489589:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572943.489594:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13630 00000400:00000200:0.0:1788572943.489600:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572943.489603:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572943.489604:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572943.489607:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489612:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572943.489614:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489615:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572943.489619:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572943.489621:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572943.489625:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572943.489628:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572943.489632:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572943.489634:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572943.489640:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489642:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572943.489653:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489659:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572943.489662:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572943.489666:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489672:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572943.489676:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572943.489680:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572943.489687:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489691:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572943.489695:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572943.489700:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489705:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572943.489710:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788572943.489714:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572943.489719:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572943.489722:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489726:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572943.489729:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572943.489733:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572943.489740:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788572943.489745:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572943.489751:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516510208/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572943.489772:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572943.489774:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572943.489777:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516510208/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572943.489785:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572943.489791:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572943.489793:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572943.489795:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572943.489804:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572943.489836:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572943.489841:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13630 00000400:00000200:0.0:1788572943.489851:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572943.489856:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572943.489860:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572943.489869:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489873:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572943.489878:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489881:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572943.489884:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572943.489886:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572943.489889:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572943.489892:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572943.489897:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572943.489900:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572943.489908:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489915:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572943.489919:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489922:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572943.489925:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572943.489931:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572943.489933:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572943.489936:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572943.489940:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572943.489943:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489946:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572943.489948:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572943.489952:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489954:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572943.489962:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788572943.489966:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572943.489969:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572943.489972:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572943.489976:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572943.489978:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572943.489982:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572943.489985:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788572943.489987:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572943.489994:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516510336/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572943.490007:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572943.490012:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572943.490015:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516510336/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572943.490039:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572943.490044:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13641) 00000100:00000001:0.0:1788572944.000959:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572944.000964:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572944.000970:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572944.000974:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572944.000980:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572944.000984:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572944.001030:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572944.001034:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572944.001038:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572945.025006:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572945.026386:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572945.026389:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572945.026393:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572945.026395:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572945.026397:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572945.026398:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572945.026399:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572945.026401:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572945.729007:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572945.729042:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572945.729046:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572946.048947:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572946.048951:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572946.048954:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572946.048958:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572946.048966:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572946.048972:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572946.048978:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572946.048981:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572946.048983:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572946.049002:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572946.049004:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572946.049005:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572946.049024:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572946.049029:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572946.049033:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572946.049050:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572946.049053:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572946.049057:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572946.752980:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572946.752984:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572946.752986:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572947.328908:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308307181, at 4308307200 00000100:00000010:0.0:1788572947.328927:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7de0. 00000100:00000001:0.0:1788572948.096985:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572948.096996:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572948.097017:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572948.097024:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572948.097031:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572948.097036:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572948.097047:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572948.097055:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572948.097066:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572948.608931:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572948.608941:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572948.608978:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572948.608982:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572948.608986:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572948.608994:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13636 00000400:00000200:0.0:1788572948.609026:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572948.609032:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572948.609036:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572948.609038:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609041:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572948.609044:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609050:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572948.609054:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572948.609056:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572948.609058:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572948.609062:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572948.609064:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572948.609066:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572948.609068:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609070:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572948.609071:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609072:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572948.609075:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572948.609077:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609079:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572948.609081:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572948.609085:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572948.609087:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.609089:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572948.609091:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572948.609098:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788572948.609101:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572948.609109:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572948.609111:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.609114:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572948.609117:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572948.609123:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572948.609131:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572948.609138:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572948.609147:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516510464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572948.609156:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572948.609162:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572948.609167:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516510464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572948.609195:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572948.609199:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572948.609203:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572948.609206:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572948.609208:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572948.609212:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572948.609215:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13636 00000400:00000200:0.0:1788572948.609220:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572948.609224:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572948.609229:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572948.609237:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609243:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572948.609246:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609253:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572948.609256:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572948.609260:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572948.609265:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572948.609270:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572948.609284:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572948.609292:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572948.609311:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609318:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572948.609327:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609335:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572948.609338:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572948.609344:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609353:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572948.609354:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572948.609361:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572948.609376:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.609379:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572948.609384:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572948.609391:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.609401:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572948.609407:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788572948.609420:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572948.609423:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572948.609426:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.609430:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572948.609433:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572948.609436:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572948.609440:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572948.609443:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572948.609450:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516510592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572948.609473:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572948.609482:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572948.609496:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516510592/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572948.609554:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572948.609560:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572948.609571:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572948.609577:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572948.609582:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572948.609594:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572948.609601:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13636 00000400:00000200:0.0:1788572948.609624:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572948.609636:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572948.609649:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572948.609660:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609672:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572948.609679:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609692:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572948.609696:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572948.609700:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572948.609711:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572948.609714:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572948.609720:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572948.609723:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572948.609726:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609729:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572948.609737:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609744:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572948.609757:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572948.609767:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572948.609771:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572948.609787:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572948.609794:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572948.609846:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.609856:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572948.609860:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572948.609869:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.609880:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572948.609885:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788572948.609894:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572948.609897:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572948.609899:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.609906:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572948.609909:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572948.609913:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572948.609925:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572948.609932:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572948.609945:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516510720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572948.610017:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572948.610020:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572948.610034:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516510720/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572948.610062:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572948.610068:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572948.610080:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572948.610082:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572948.610087:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572948.610097:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572948.610105:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13636 00000400:00000200:0.0:1788572948.610115:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572948.610126:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572948.610131:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572948.610143:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.610146:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572948.610149:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.610151:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572948.610157:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572948.610161:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572948.610170:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572948.610175:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572948.610183:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572948.610189:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572948.610202:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.610208:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572948.610213:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.610217:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572948.610223:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572948.610229:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572948.610236:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572948.610238:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572948.610248:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572948.610256:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.610261:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572948.610266:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572948.610269:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.610275:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572948.610285:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788572948.610290:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572948.610294:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572948.610297:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.610300:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572948.610303:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572948.610305:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572948.610312:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788572948.610321:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572948.610330:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516510848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572948.610353:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572948.610360:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572948.610370:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516510848/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572948.610390:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572948.610398:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572948.610411:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572948.610420:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572948.610427:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572948.610431:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572948.610433:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13636 00000400:00000200:0.0:1788572948.610440:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572948.610454:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572948.610458:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572948.610470:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.610474:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572948.610476:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.610479:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572948.610482:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572948.610487:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572948.610499:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572948.610510:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572948.610525:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572948.610535:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572948.610552:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.610558:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572948.610573:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572948.610580:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572948.610593:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572948.610606:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572948.610616:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572948.610624:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572948.610634:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572948.610642:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.610651:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572948.610655:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572948.610660:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.610666:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572948.610679:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788572948.610687:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572948.610695:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572948.610704:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572948.610713:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572948.610721:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572948.610726:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572948.610737:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788572948.610746:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572948.610762:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516510976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572948.610803:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572948.610830:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572948.610844:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516510976/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572948.610884:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572948.610893:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13646) 00000100:00000001:0.0:1788572949.120941:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572949.120949:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572949.120954:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572949.120975:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572949.120980:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572949.120984:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572949.120988:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572949.120991:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572949.121006:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572950.144937:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572950.144943:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572950.144947:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572950.144953:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572950.144960:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572950.144965:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572950.146293:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572950.146296:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572950.146307:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572950.849020:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572950.849024:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572950.849027:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788572951.872983:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572951.873007:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572951.873010:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788572952.448887:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308312302, at 4308312320 00000100:00000010:0.0:1788572952.448905:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea78a0. 00000100:00000001:0.0:1788572953.216941:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572953.216947:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572953.216951:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572953.216957:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572953.216962:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572953.216967:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572953.216972:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572953.216975:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572953.216978:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572953.728933:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572953.728944:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572953.728948:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572953.728955:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572953.728998:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572953.729006:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13641 00000400:00000200:0.0:1788572953.729016:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572953.729023:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572953.729026:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572953.729029:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729031:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572953.729033:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729035:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572953.729039:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572953.729041:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572953.729043:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572953.729047:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572953.729049:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572953.729051:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572953.729053:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729055:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572953.729056:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729057:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572953.729060:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572953.729063:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729064:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572953.729067:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572953.729070:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572953.729073:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.729074:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572953.729076:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572953.729083:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788572953.729086:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572953.729089:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572953.729090:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.729099:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572953.729102:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572953.729108:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572953.729117:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788572953.729128:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572953.729141:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516511104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572953.729155:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572953.729162:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572953.729167:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516511104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572953.729192:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572953.729196:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572953.729200:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572953.729202:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572953.729205:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572953.729208:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572953.729212:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13641 00000400:00000200:0.0:1788572953.729218:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572953.729225:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572953.729229:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572953.729234:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729238:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572953.729244:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729249:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572953.729254:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572953.729256:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572953.729259:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572953.729262:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572953.729265:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572953.729268:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572953.729286:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729291:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572953.729296:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729299:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572953.729304:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572953.729310:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729314:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572953.729316:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572953.729323:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572953.729329:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.729334:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572953.729339:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572953.729342:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.729344:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572953.729352:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788572953.729354:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572953.729357:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572953.729360:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.729363:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572953.729365:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572953.729369:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572953.729374:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788572953.729381:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572953.729391:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516511232/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572953.729403:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572953.729406:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572953.729414:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516511232/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572953.729423:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572953.729427:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572953.729438:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572953.729442:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572953.729444:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572953.729449:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572953.729454:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13641 00000400:00000200:0.0:1788572953.729461:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572953.729465:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572953.729470:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572953.729473:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729475:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572953.729476:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729478:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572953.729479:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572953.729482:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572953.729485:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572953.729488:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572953.729492:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572953.729497:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572953.729503:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729505:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572953.729507:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729509:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572953.729513:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572953.729519:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729525:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572953.729527:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572953.729529:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572953.729532:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.729535:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572953.729536:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572953.729547:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.729552:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572953.729557:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788572953.729560:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572953.729565:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572953.729569:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.729572:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572953.729576:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572953.729579:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572953.729584:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788572953.729587:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572953.729593:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516511360/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572953.729602:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572953.729607:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572953.729611:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516511360/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572953.729623:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572953.729625:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572953.729630:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572953.729633:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572953.729636:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572953.729641:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572953.729646:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13641 00000400:00000200:0.0:1788572953.729651:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572953.729657:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572953.729661:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572953.729666:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729671:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572953.729675:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729684:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572953.729687:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572953.729690:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572953.729694:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572953.729697:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572953.729700:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572953.729704:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572953.729710:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729713:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572953.729716:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729718:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572953.729720:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572953.729722:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729724:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572953.729729:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572953.729732:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572953.729734:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.729739:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572953.729742:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572953.729746:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.729748:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572953.729754:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788572953.729756:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572953.729758:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572953.729758:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.729763:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572953.729765:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572953.729768:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572953.729773:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788572953.729779:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572953.729788:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516511488/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572953.729841:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572953.729844:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572953.729852:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516511488/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572953.729871:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572953.729874:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572953.729878:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572953.729882:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572953.729884:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572953.729889:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572953.729894:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13641 00000400:00000200:0.0:1788572953.729901:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572953.729906:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572953.729911:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572953.729918:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729923:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572953.729928:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729933:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572953.729936:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572953.729938:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572953.729940:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572953.729943:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572953.729949:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572953.729953:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572953.729960:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729966:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572953.729971:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572953.729973:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572953.729997:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572953.730003:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572953.730007:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572953.730010:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572953.730013:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572953.730016:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.730018:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572953.730019:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572953.730022:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.730027:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572953.730031:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788572953.730035:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572953.730039:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572953.730042:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572953.730050:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572953.730053:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572953.730056:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572953.730063:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788572953.730068:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572953.730076:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516511616/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572953.730099:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572953.730102:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572953.730108:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516511616/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572953.730119:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572953.730129:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13651) 00000100:00000001:0.0:1788572955.265006:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572955.265016:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572955.265033:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572955.265046:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572955.265053:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572955.265073:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572955.265080:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572955.265085:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572955.265089:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572955.969004:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572955.969011:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572955.969017:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572956.289073:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572956.289082:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572956.289091:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572956.289099:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572956.289103:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572956.289105:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572956.289108:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572956.289110:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572956.289114:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572956.993084:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572956.993092:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572956.993097:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572957.313006:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572957.313013:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572957.313020:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572957.313028:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572957.313033:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572957.313038:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572957.313057:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572957.313060:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572957.313066:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572957.568883:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308317421, at 4308317440 00000100:00000010:0.0:1788572957.568901:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea74e0. 00000100:00000001:0.0:1788572958.336919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572958.336928:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572958.336934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572958.336943:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572958.336949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572958.336956:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572958.336987:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572958.336999:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572958.337003:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572958.848946:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572958.848956:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572958.848958:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572958.848973:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572958.848979:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572958.848985:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13646 00000400:00000200:0.0:1788572958.848997:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572958.849006:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572958.849009:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572958.849011:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849014:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572958.849016:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849019:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572958.849022:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572958.849024:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572958.849026:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572958.849029:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572958.849031:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572958.849033:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572958.849037:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849038:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572958.849041:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849044:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572958.849047:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572958.849051:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849053:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572958.849056:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572958.849059:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572958.849062:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849080:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572958.849082:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572958.849088:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788572958.849092:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572958.849095:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572958.849097:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849100:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572958.849103:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572958.849107:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572958.849113:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788572958.849119:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572958.849127:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516511744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572958.849135:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572958.849139:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572958.849144:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516511744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572958.849162:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572958.849165:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572958.849168:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572958.849169:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572958.849170:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572958.849172:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572958.849174:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13646 00000400:00000200:0.0:1788572958.849179:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572958.849183:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572958.849186:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572958.849190:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849194:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572958.849196:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849200:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572958.849208:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572958.849212:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572958.849215:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572958.849217:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572958.849222:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572958.849224:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572958.849231:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849235:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572958.849239:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849244:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572958.849247:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572958.849253:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849256:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572958.849258:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572958.849266:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572958.849276:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849281:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572958.849287:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572958.849292:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849298:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572958.849305:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788572958.849309:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572958.849315:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572958.849317:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849322:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572958.849325:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572958.849327:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572958.849334:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788572958.849338:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572958.849345:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516511872/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572958.849355:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572958.849365:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572958.849375:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516511872/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572958.849392:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572958.849395:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572958.849399:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572958.849404:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572958.849407:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572958.849409:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572958.849413:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13646 00000400:00000200:0.0:1788572958.849420:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572958.849427:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572958.849430:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572958.849436:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849442:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572958.849448:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849453:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572958.849458:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572958.849459:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572958.849464:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572958.849468:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572958.849471:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572958.849473:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572958.849481:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849484:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572958.849490:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849494:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572958.849496:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572958.849502:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849511:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572958.849514:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572958.849518:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572958.849524:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849526:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572958.849529:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572958.849533:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849536:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572958.849540:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788572958.849546:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572958.849548:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572958.849550:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849554:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572958.849555:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572958.849559:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572958.849562:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572958.849566:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572958.849574:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516512000/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572958.849584:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572958.849587:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572958.849595:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516512000/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572958.849604:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572958.849608:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572958.849610:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572958.849614:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572958.849616:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572958.849620:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572958.849624:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13646 00000400:00000200:0.0:1788572958.849632:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572958.849642:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572958.849645:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572958.849647:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849649:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572958.849650:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849651:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572958.849652:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572958.849656:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572958.849658:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572958.849660:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572958.849661:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572958.849663:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572958.849669:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849673:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572958.849678:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849682:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572958.849686:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572958.849692:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849699:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572958.849702:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572958.849708:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572958.849715:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849719:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572958.849724:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572958.849729:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849733:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572958.849739:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788572958.849744:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572958.849758:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572958.849762:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849766:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572958.849774:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572958.849777:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572958.849783:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788572958.849788:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572958.849797:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516512128/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572958.849806:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572958.849826:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572958.849830:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516512128/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572958.849836:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572958.849838:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572958.849842:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572958.849845:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572958.849847:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572958.849850:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572958.849855:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13646 00000400:00000200:0.0:1788572958.849861:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572958.849866:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572958.849869:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572958.849874:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849878:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572958.849882:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849887:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572958.849891:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572958.849894:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572958.849897:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572958.849900:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572958.849904:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572958.849909:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572958.849920:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849925:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572958.849927:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849930:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572958.849936:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572958.849941:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572958.849943:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572958.849944:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572958.849946:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572958.849948:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849949:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572958.849952:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572958.849957:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849961:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572958.849979:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788572958.849983:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572958.849989:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572958.849992:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572958.849997:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572958.849999:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572958.850001:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572958.850004:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788572958.850008:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572958.850018:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516512256/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572958.850029:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572958.850034:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572958.850042:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516512256/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572958.850055:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572958.850059:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13656) 00000100:00000001:0.0:1788572959.360891:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572959.360900:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572959.360904:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572959.360913:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572959.360921:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572959.360928:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572959.360947:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572959.360948:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572959.360951:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572959.360993:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572959.360997:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572959.360999:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572959.361002:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572959.361005:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572959.361015:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572959.361017:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572959.361018:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572959.361020:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572960.384937:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572960.384971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572960.384973:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572960.384979:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572960.384984:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572960.384988:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572960.385011:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572960.385016:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572960.385019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572961.089074:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572961.089077:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572961.089080:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572961.408895:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572961.408899:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572961.408902:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572961.408909:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572961.408917:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572961.408923:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572961.408941:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572961.408944:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572961.408969:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572962.112973:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572962.112976:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572962.112979:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572962.432939:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572962.432948:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572962.432967:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572962.432989:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572962.432996:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572962.433001:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572962.433016:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572962.433019:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572962.433022:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572962.688923:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308322541, at 4308322560 00000100:00000010:0.0:1788572962.688939:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7900. 00000100:00000001:0.0:1788572963.456937:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572963.456958:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572963.456961:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572963.456968:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572963.456976:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572963.456982:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572963.456989:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572963.456992:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572963.456996:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.457016:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572963.457019:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572963.457021:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572963.457025:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572963.457029:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572963.457033:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572963.457038:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572963.457041:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572963.457044:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572963.968891:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572963.968905:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788572963.968909:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572963.968930:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572963.968935:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572963.968943:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13651 00000400:00000200:0.0:1788572963.968964:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572963.968970:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572963.968973:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572963.968975:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.968978:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572963.968980:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.968982:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572963.968985:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572963.968986:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788572963.968988:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572963.968991:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788572963.968993:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788572963.968995:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788572963.968997:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.968998:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572963.968999:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969000:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572963.969002:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788572963.969005:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969006:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572963.969009:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572963.969011:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572963.969013:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969015:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788572963.969017:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572963.969021:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788572963.969024:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788572963.969027:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572963.969032:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969035:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572963.969037:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572963.969041:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572963.969048:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788572963.969053:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572963.969060:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516512384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572963.969068:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572963.969071:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572963.969075:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516512384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572963.969098:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572963.969101:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572963.969105:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788572963.969107:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572963.969109:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572963.969112:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572963.969116:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13651 00000400:00000200:0.0:1788572963.969122:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572963.969128:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572963.969133:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572963.969139:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969143:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572963.969146:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969151:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572963.969157:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572963.969159:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788572963.969162:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572963.969166:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788572963.969169:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788572963.969178:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788572963.969182:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969186:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572963.969191:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969196:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572963.969199:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788572963.969204:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969209:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572963.969215:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572963.969223:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572963.969227:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969230:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572963.969234:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572963.969239:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969242:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572963.969249:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788572963.969252:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572963.969257:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572963.969258:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969259:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572963.969261:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572963.969265:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572963.969271:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788572963.969275:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572963.969284:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516512512/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572963.969305:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572963.969309:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572963.969317:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516512512/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572963.969333:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572963.969342:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572963.969348:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788572963.969350:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572963.969354:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572963.969359:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572963.969364:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13651 00000400:00000200:0.0:1788572963.969371:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572963.969378:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572963.969383:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572963.969391:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969394:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572963.969396:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969398:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572963.969400:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572963.969401:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788572963.969402:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572963.969403:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788572963.969405:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788572963.969406:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788572963.969408:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969410:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572963.969411:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969414:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572963.969417:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788572963.969422:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969424:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572963.969427:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572963.969432:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572963.969437:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969440:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572963.969448:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788572963.969452:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969455:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572963.969460:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788572963.969465:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788572963.969471:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572963.969475:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969479:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572963.969482:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572963.969485:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572963.969490:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788572963.969493:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572963.969500:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516512640/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572963.969508:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572963.969510:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572963.969517:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516512640/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572963.969531:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572963.969534:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572963.969540:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788572963.969542:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572963.969545:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572963.969548:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572963.969551:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13651 00000400:00000200:0.0:1788572963.969559:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572963.969563:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572963.969566:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572963.969573:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969579:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572963.969585:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969594:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572963.969599:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572963.969602:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788572963.969606:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572963.969610:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788572963.969616:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788572963.969621:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788572963.969627:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969631:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572963.969634:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969636:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572963.969638:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788572963.969641:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969646:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572963.969648:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572963.969652:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572963.969656:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969663:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572963.969669:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572963.969674:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969679:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572963.969684:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788572963.969691:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=4 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572963.969694:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572963.969698:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969701:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572963.969705:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572963.969708:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572963.969716:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788572963.969721:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572963.969729:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516512768/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572963.969768:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572963.969772:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572963.969780:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516512768/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572963.969801:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788572963.969805:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788572963.969827:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788572963.969828:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788572963.969830:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788572963.969835:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788572963.969840:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13651 00000400:00000200:0.0:1788572963.969848:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572963.969854:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788572963.969856:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788572963.969862:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969864:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572963.969867:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969871:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788572963.969876:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788572963.969879:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788572963.969882:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788572963.969886:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788572963.969892:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788572963.969898:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788572963.969909:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969916:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788572963.969922:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969930:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788572963.969954:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788572963.969961:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788572963.969966:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788572963.969969:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788572963.969973:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788572963.969978:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969981:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788572963.969984:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788572963.969987:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.969991:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788572963.969996:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788572963.969998:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788572963.970001:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788572963.970004:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572963.970007:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788572963.970009:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788572963.970011:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788572963.970017:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788572963.970023:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572963.970030:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516512896/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788572963.970045:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788572963.970049:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572963.970054:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516512896/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572963.970063:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572963.970068:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13661) 00000020:00000001:0.0:1788572964.161197:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572964.161202:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572964.161207:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572964.480931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572964.480939:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572964.480979:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572964.480983:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572964.480988:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572964.480992:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572964.480995:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572964.480998:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572964.481001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572965.185050:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572965.185054:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572965.185056:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572965.504925:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572965.504935:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572965.504954:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572965.504961:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572965.504966:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572965.504971:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572965.504975:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572965.504979:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572965.504983:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572965.505016:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572965.505020:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572965.505022:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572965.505025:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572965.505029:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572965.505032:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572965.505034:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572965.505037:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572965.505039:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572966.209032:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572966.209035:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572966.209038:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572966.528922:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572966.528928:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572966.528933:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572966.528951:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572966.528958:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572966.528964:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572966.528977:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572966.528997:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572966.529003:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572967.552921:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572967.552926:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572967.552930:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572967.552950:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572967.552955:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572967.552960:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572967.552965:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572967.552969:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572967.552978:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572967.552997:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572967.553002:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572967.553005:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572967.553012:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572967.553016:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572967.553020:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572967.553022:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572967.553024:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572967.553026:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572967.808918:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308327661, at 4308327680 00000100:00000010:0.0:1788572967.808950:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea71e0. 00000020:00000001:0.0:1788572968.257021:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572968.257025:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572968.257028:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572969.601029:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572969.601036:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572969.601042:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572969.601052:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572969.601069:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572969.601073:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572969.601079:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572969.601090:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572969.601096:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572970.305072:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572970.305075:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572970.305078:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572970.624903:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572970.624907:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572970.624910:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572970.624917:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572970.624938:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572970.624944:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572970.625142:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572970.625146:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572970.625149:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572971.329043:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572971.329047:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572971.329051:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572971.648954:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572971.648962:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572971.648966:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572971.648972:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572971.648980:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572971.648985:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572971.649008:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572971.649013:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572971.649018:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788572972.161128:0:2388:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788572972.161178:0:2388:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788572972.161187:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572972.161191:0:2388:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788572972.161196:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788572972.161201:0:2388:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788572972.161206:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572972.161210:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793700. 00000400:00000200:0.0:1788572972.161217:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572972.161221:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572972.161246:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc110 00000400:00000010:0.0:1788572972.161250:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc110. 00000100:00000001:0.0:1788572972.161256:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572972.161266:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac820700 x1875440516513152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572972.161305:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572972.161311:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516513152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788572994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572972.161320:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572972.161322:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572972.161335:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572972.161338:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793000. 00000400:00000200:0.0:1788572972.161342:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572972.161348:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572972.161354:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc330 00000400:00000010:0.0:1788572972.161358:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc330. 00000100:00000001:0.0:1788572972.161361:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572972.161366:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac822d80 x1875440516513280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572972.161379:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572972.161387:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516513280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572972.161415:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572972.161418:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572972.161426:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572972.161432:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793a00. 00000400:00000200:0.0:1788572972.161436:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572972.161443:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572972.161448:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcbb0 00000400:00000010:0.0:1788572972.161452:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcbb0. 00000100:00000001:0.0:1788572972.161459:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572972.161466:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac823480 x1875440516513408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572972.161487:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572972.161496:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516513408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788572994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572972.161515:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572972.161520:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572972.161531:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572972.161536:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793800. 00000400:00000200:0.0:1788572972.161543:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572972.161552:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572972.161558:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fccc0 00000400:00000010:0.0:1788572972.161562:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fccc0. 00000100:00000001:0.0:1788572972.161566:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572972.161573:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac822300 x1875440516513536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572972.161594:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572972.161600:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516513536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572972.161616:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572972.161621:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788572972.161631:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788572972.161637:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793300. 00000400:00000200:0.0:1788572972.161643:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788572972.161652:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788572972.161658:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc550 00000400:00000010:0.0:1788572972.161662:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc550. 00000100:00000001:0.0:1788572972.161667:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788572972.161677:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac821f80 x1875440516513664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572972.161695:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788572972.161703:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516513664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788572994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788572972.161712:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572972.161713:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788572972.672902:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572972.672907:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572972.672910:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572972.672938:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572972.672944:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572972.672965:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572972.672971:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572972.672975:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572972.672981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572972.673017:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572972.673020:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572972.673022:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572972.673024:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572972.673028:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572972.673031:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572972.673033:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572972.673035:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572972.673036:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572972.928894:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308332782, at 4308332800 00000100:00000010:0.0:1788572972.928935:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7840. 00000020:00000001:0.0:1788572973.377014:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572973.377017:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572973.377020:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572973.696963:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572973.696969:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572973.696974:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572973.696982:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572973.696989:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572973.696995:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572973.697000:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572973.697003:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572973.697007:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572973.697022:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572973.697027:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572973.697029:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572973.697036:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572973.697039:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572973.697058:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572973.697060:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572973.697061:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572973.697063:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572974.720897:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572974.720905:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572974.720921:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572974.720929:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572974.720938:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572974.720945:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572974.720953:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572974.720957:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572974.720963:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572974.720984:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572974.720989:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572974.720991:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572974.720994:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572974.720997:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572974.721001:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572974.721004:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572974.721006:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572974.721009:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572975.425034:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572975.425037:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572975.425040:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572975.744945:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572975.744953:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572975.744958:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572975.744966:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572975.744975:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572975.744981:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572975.744987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572975.744989:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572975.744992:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572976.449011:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572976.449016:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572976.449020:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572976.768914:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572976.768940:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572976.768944:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572976.768951:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572976.768957:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572976.768961:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572976.768985:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572976.768999:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572976.769005:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572977.792924:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572977.792938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572977.792943:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572977.792946:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572977.792951:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572977.792955:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572977.792959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572977.792963:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572977.792967:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572977.793008:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572977.793014:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572977.793018:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572977.793022:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572977.793025:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572977.793027:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572977.793029:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572977.793031:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572977.793033:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572978.048950:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308337902, at 4308337920 00000100:00000010:0.0:1788572978.048979:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7d80. 00000020:00000001:0.0:1788572978.497062:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572978.497066:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572978.497070:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572978.816936:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572978.816943:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572978.816947:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572978.816954:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572978.816961:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572978.816966:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572978.816993:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572978.816997:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572978.817001:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572979.840933:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572979.840942:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572979.840946:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572979.840952:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572979.840956:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572979.840962:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572979.840975:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572979.840979:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572979.840982:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572980.545062:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572980.545066:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572980.545070:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572980.864923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572980.864934:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572980.864938:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572980.864947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572980.864962:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572980.864968:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572980.865943:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572980.865946:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572980.865949:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572981.569017:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572981.569021:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572981.569025:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572981.888934:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572981.888944:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572981.888949:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572981.888958:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572981.888967:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572981.888973:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572981.888998:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572981.889002:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572981.889005:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572982.912948:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572982.912957:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572982.912983:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572982.913005:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572982.913009:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572982.913011:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572982.913013:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572982.913015:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572982.913020:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572983.168913:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308343021, at 4308343040 00000100:00000010:0.0:1788572983.168931:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916749c0. 00000020:00000001:0.0:1788572983.617084:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572983.617089:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572983.617093:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572983.936921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572983.936930:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572983.936934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572983.936942:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572983.936950:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572983.936955:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572983.936959:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572983.936963:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572983.936967:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572983.936999:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572983.937002:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572983.937004:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572983.937006:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788572983.937008:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572983.937011:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572983.937012:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572983.937014:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572983.937016:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572984.960925:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572984.960933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572984.960939:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572984.960945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572984.960953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572984.960957:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572984.960975:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572984.960992:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572984.960996:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572985.984980:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572985.985008:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572985.985013:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572985.985021:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572985.985029:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572985.985035:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572985.985043:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572985.985047:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572985.985050:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572985.985233:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572985.985235:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572985.985236:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572985.985237:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572985.985239:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572985.985240:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572985.985241:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572985.985242:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572985.985244:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572986.689109:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572986.689113:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572986.689116:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572987.008929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572987.008939:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572987.008944:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572987.008951:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572987.008957:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572987.008961:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572987.009098:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572987.009102:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572987.009107:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572988.032948:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572988.032954:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572988.032958:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572988.032966:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572988.032973:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572988.032978:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572988.032991:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572988.032994:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572988.032998:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572988.033212:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572988.033216:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572988.033217:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572988.033220:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572988.033222:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572988.033224:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572988.033226:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572988.033228:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572988.033230:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572988.288939:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308348141, at 4308348160 00000100:00000010:0.0:1788572988.288954:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c00. 00000020:00000001:0.0:1788572988.737025:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572988.737028:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572988.737031:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572989.056996:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572989.057002:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572989.057007:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572989.057014:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572989.057021:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572989.057025:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572989.057029:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572989.057032:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572989.057034:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572990.080909:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572990.080919:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572990.080922:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572990.080930:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572990.080939:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572990.080946:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572990.080968:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572990.080973:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572990.080979:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572991.104942:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572991.104949:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572991.104969:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572991.104973:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572991.104977:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572991.104982:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572991.105118:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572991.105121:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572991.105124:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788572991.809014:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572991.809017:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572991.809019:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572992.128897:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572992.128905:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572992.128908:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572992.128916:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572992.128923:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572992.128929:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572992.128936:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572992.128940:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572992.128944:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572992.128975:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572992.128978:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572992.128980:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572992.128983:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572992.128989:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572992.128991:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572992.129001:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572992.129004:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572992.129006:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572993.152952:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572993.152960:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572993.152963:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572993.152970:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572993.152977:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572993.152981:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572993.152986:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572993.152989:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572993.152992:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572993.153012:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572993.153023:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572993.153026:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572993.153031:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572993.153036:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572993.153040:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572993.153043:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572993.153044:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572993.153046:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572993.408944:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308353261, at 4308353280 00000100:00000010:0.0:1788572993.408961:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674720. 00000020:00000001:0.0:1788572993.857087:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572993.857091:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572993.857095:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572994.176940:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572994.176947:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572994.176953:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572994.176961:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788572994.176969:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572994.176974:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572994.176988:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572994.176992:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572994.176996:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572994.177012:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572994.177016:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572994.177018:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572994.177019:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572994.177022:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572994.177024:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572994.177026:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572994.177027:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572994.177029:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572995.200947:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572995.200953:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572995.200958:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572995.200967:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572995.200974:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572995.200979:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572995.201008:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572995.201011:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572995.201015:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572996.224926:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572996.224933:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572996.224936:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572996.224941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788572996.224947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572996.224953:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572996.224959:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572996.224962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572996.224968:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572996.224998:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572996.225002:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572996.225006:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572996.225008:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788572996.225010:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572996.225012:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572996.225014:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572996.225016:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572996.225018:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788572996.736934:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788572996.736946:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788572996.736952:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788572996.737097:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572996.737111:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788572996.737117:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788572996.737121:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788572996.929013:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572996.929017:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572996.929021:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572997.249090:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572997.249095:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572997.249099:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572997.249104:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572997.249107:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572997.249111:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572997.249125:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572997.249128:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572997.249131:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788572998.272931:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572998.272941:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572998.272946:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572998.272954:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572998.272961:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572998.272967:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572998.272973:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572998.272977:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572998.272979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788572998.528940:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308358381, at 4308358400 00000100:00000010:0.0:1788572998.528956:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674360. 00000020:00000001:0.0:1788572998.977062:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788572998.977066:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788572998.977070:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788572999.296954:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788572999.296960:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788572999.296965:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788572999.296969:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788572999.296973:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788572999.296980:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572999.296989:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788572999.296993:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788572999.296997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573000.320890:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573000.320900:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573000.320904:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573000.320911:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573000.320917:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573000.320922:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573000.320938:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573000.320944:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573000.320954:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573001.344913:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573001.344923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573001.344944:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573001.344949:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573001.344957:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573001.344960:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573001.344972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573001.344976:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573001.344981:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573002.049021:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573002.049024:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573002.049027:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573002.368874:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573002.368883:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573002.368886:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573002.368893:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573002.368901:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573002.368908:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573002.368915:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573002.368922:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573002.368928:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573002.368963:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573002.368967:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573002.368969:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573002.368971:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573002.368974:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573002.368976:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573002.368978:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573002.368980:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573002.368982:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573003.392944:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573003.392951:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573003.392955:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573003.392962:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573003.392969:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573003.392974:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573003.392979:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573003.392981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573003.392984:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573003.648921:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308363501, at 4308363520 00000100:00000010:0.0:1788573003.648945:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674d80. 00000020:00000001:0.0:1788573004.097060:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573004.097063:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573004.097066:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573004.416927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573004.416936:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573004.416942:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573004.416952:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573004.416959:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573004.416963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573004.416967:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573004.416972:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573004.416975:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573004.417056:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573004.417059:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573004.417061:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573004.417063:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573004.417065:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573004.417067:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573004.417069:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573004.417070:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573004.417073:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573005.440913:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573005.440920:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573005.440924:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573005.440928:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573005.440937:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573005.440942:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573005.440962:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573005.440965:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573005.440968:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573005.440986:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573005.440990:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573005.440991:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573005.440994:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573005.440996:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573005.440998:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573005.441014:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573005.441015:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573005.441017:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573006.464923:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573006.464933:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573006.464938:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573006.464947:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573006.464954:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573006.464961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573006.464982:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573006.464984:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573006.464987:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573006.465006:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573006.465009:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573006.465010:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573006.465013:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573006.465016:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573006.465019:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573006.465022:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573006.465023:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573006.465025:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573007.169076:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573007.169079:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573007.169082:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573007.488898:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573007.488906:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573007.488912:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573007.488920:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573007.488927:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573007.488935:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573007.488950:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573007.488954:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573007.488958:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573008.512937:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573008.512945:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573008.512950:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573008.512963:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573008.512976:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573008.513000:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573008.513012:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573008.513017:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573008.513029:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573008.768939:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308368621, at 4308368640 00000100:00000010:0.0:1788573008.768962:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674780. 00000020:00000001:0.0:1788573009.217055:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573009.217060:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573009.217063:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573009.536907:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573009.536919:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573009.536923:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573009.536933:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573009.536941:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573009.536945:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573009.536951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573009.536955:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573009.536961:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573009.537001:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573009.537006:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573009.537014:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573009.537024:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573009.537043:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573009.537045:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573009.537048:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573009.537049:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573009.537051:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573010.560946:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573010.560954:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573010.560958:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573010.560967:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573010.560976:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573010.560981:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573010.560992:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573010.560996:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573010.561011:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573010.561058:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573010.561071:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573010.561072:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573010.561074:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573010.561077:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573010.561080:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573010.561082:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573010.561084:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573010.561086:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573011.584952:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573011.584962:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573011.584966:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573011.584973:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573011.584978:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573011.584983:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573011.584994:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573011.585001:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573011.585010:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573012.289170:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573012.289173:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573012.289175:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573012.608921:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573012.608927:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573012.608930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573012.608938:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573012.608949:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573012.608954:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573012.608981:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573012.608985:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573012.608988:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573012.609003:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573012.609005:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573012.609007:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573012.609008:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573012.609011:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573012.609012:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573012.609014:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573012.609016:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573012.609018:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573013.632919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573013.632926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573013.632930:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573013.632942:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573013.632949:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573013.632954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573013.632972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573013.632976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573013.632980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573013.888881:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308373741, at 4308373760 00000100:00000010:0.0:1788573013.888894:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7000. 00000020:00000001:0.0:1788573014.337033:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573014.337037:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573014.337041:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573014.656915:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573014.656925:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573014.656930:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573014.656947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573014.656955:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573014.656962:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.656981:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573014.656984:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573014.656988:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573015.680907:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573015.680924:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573015.680927:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573015.680934:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573015.680940:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573015.680944:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573015.680951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573015.680954:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573015.680960:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573015.680992:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573015.680997:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573015.681001:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573015.681005:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573015.681010:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573015.681024:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573015.681027:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573015.681029:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573015.681031:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573016.704913:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573016.704921:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573016.704925:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573016.704934:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573016.704942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573016.704947:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573016.704951:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573016.704956:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573016.704961:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573016.704975:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573016.704979:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573016.704982:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573016.704987:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573016.704991:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573016.704994:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573016.704995:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573016.704996:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573016.704998:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573017.409004:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573017.409007:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573017.409010:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573017.728894:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573017.728903:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573017.728907:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573017.728914:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573017.728919:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573017.728924:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573017.728937:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573017.728941:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573017.728946:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573017.728964:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573017.728968:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573017.728970:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573017.728974:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573017.728987:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573017.728990:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573017.728993:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573017.728995:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573017.728997:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573018.752930:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573018.752940:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573018.752944:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573018.752953:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573018.752959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573018.752965:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573018.752969:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573018.752973:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573018.752979:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573018.752993:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573018.752998:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573018.753001:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573018.753004:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573018.753011:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573018.753015:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573018.753020:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573018.753022:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573018.753024:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573019.008908:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308378861, at 4308378880 00000100:00000010:0.0:1788573019.008928:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674840. 00000020:00000001:0.0:1788573019.457070:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573019.457079:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573019.457083:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573019.776904:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573019.776908:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573019.776912:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573019.776919:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573019.776924:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573019.776931:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.776937:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573019.776942:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573019.776945:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573020.800943:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573020.800949:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573020.800953:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573020.800961:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573020.800967:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573020.800970:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573020.801019:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573020.801024:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573020.801032:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573021.824948:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573021.824955:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573021.824958:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573021.824962:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573021.824965:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573021.824967:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573021.824969:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573021.824971:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573021.824973:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573021.824989:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573021.824994:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573021.824996:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573021.825000:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573021.825005:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573021.825009:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573021.825014:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573021.825018:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573021.825020:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573021.825093:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573021.825095:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573021.825096:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573021.825098:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573021.825100:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573021.825101:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573021.825120:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573021.825124:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573021.825126:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573022.528984:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573022.528987:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573022.529005:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573022.848913:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573022.848920:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573022.848925:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573022.848932:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573022.848939:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573022.848943:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573022.848951:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573022.848956:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573022.848959:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573022.848994:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573022.848996:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573022.848997:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573022.848999:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573022.849002:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573022.849004:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573022.850655:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573022.850658:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573022.850665:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573023.872924:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573023.872930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573023.872935:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573023.872941:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573023.872948:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573023.872953:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573023.872965:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573023.872969:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573023.872972:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573024.128876:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308383981, at 4308384000 00000100:00000010:0.0:1788573024.128895:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674ea0. 00000020:00000001:0.0:1788573024.576995:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573024.577000:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573024.577012:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573024.896929:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573024.896937:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573024.896941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573024.896949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573024.896973:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573024.896978:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573024.896985:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573024.896989:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573024.896995:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573025.408892:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573025.408901:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573025.408904:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573025.408908:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573025.408912:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573025.408919:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13712 00000400:00000200:0.0:1788573025.408926:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573025.408932:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573025.408934:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573025.408937:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.408940:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.408943:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.408945:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573025.408949:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573025.408951:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573025.408953:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573025.408956:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573025.408958:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573025.408960:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573025.408963:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.408965:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.408967:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.408968:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573025.408971:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573025.408973:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573025.408975:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573025.408992:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573025.408995:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573025.408997:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.408999:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573025.409000:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573025.409005:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573025.409008:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573025.409012:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573025.409013:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409016:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573025.409018:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573025.409022:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573025.409028:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573025.409033:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573025.409041:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516520192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573025.409052:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573025.409056:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573025.409062:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516520192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573025.409084:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573025.409087:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573025.409092:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573025.409093:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573025.409095:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573025.409098:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573025.409101:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13712 00000400:00000200:0.0:1788573025.409108:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573025.409112:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573025.409114:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573025.409118:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409122:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.409133:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409136:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573025.409141:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573025.409143:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573025.409146:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573025.409150:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573025.409154:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573025.409160:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573025.409165:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409170:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.409174:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409178:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573025.409184:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573025.409190:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409195:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573025.409198:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573025.409205:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573025.409210:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409213:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573025.409217:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573025.409221:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409224:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573025.409229:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3100. 00000020:00000040:0.0:1788573025.409231:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573025.409235:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573025.409236:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409237:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573025.409239:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573025.409243:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573025.409247:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573025.409251:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573025.409263:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3100 x1875440516520320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573025.409287:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573025.409290:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573025.409298:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053a4b590] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3100 x1875440516520320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573025.409311:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573025.409315:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573025.409319:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573025.409320:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573025.409321:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573025.409325:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573025.409329:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13712 00000400:00000200:0.0:1788573025.409366:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573025.409371:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573025.409374:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573025.409380:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409385:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.409389:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409392:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573025.409396:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573025.409397:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573025.409398:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573025.409400:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573025.409405:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573025.409410:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573025.409415:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409419:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.409422:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409425:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573025.409448:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573025.409454:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409460:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573025.409463:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573025.409468:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573025.409471:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409477:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573025.409479:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573025.409484:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409488:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573025.409492:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573025.409497:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573025.409500:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573025.409502:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409506:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573025.409508:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573025.409511:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573025.409516:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573025.409520:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573025.409528:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516520448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573025.409535:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573025.409538:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573025.409544:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516520448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573025.409552:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573025.409554:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573025.409557:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573025.409560:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573025.409562:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573025.409564:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573025.409574:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13712 00000400:00000200:0.0:1788573025.409581:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573025.409584:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573025.409589:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573025.409593:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409598:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.409601:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409605:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573025.409608:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573025.409609:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573025.409612:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573025.409615:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573025.409620:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573025.409622:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573025.409626:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409628:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.409632:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409634:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573025.409640:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573025.409645:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409651:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573025.409655:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573025.409659:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573025.409662:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409665:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573025.409668:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573025.409672:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409677:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573025.409680:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573025.409685:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573025.409690:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573025.409696:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409700:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573025.409703:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573025.409706:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573025.409711:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573025.409717:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573025.409725:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516520576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573025.409737:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573025.409741:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573025.409750:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516520576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573025.409765:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573025.409770:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573025.409774:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573025.409776:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573025.409777:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573025.409781:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573025.409786:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13712 00000400:00000200:0.0:1788573025.409793:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573025.409797:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573025.409800:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573025.409803:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409806:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.409826:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409828:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573025.409829:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573025.409830:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573025.409831:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573025.409833:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573025.409834:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573025.409850:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573025.409858:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409863:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573025.409869:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409875:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573025.409880:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573025.409886:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573025.409889:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573025.409894:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573025.409898:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573025.409902:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409906:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573025.409909:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573025.409914:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409918:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573025.409922:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573025.409928:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573025.409935:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573025.409939:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573025.409942:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573025.409946:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573025.409949:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573025.409955:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573025.409968:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573025.409977:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516520704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573025.409985:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573025.409986:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573025.409992:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516520704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573025.410004:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573025.410017:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13722) 00000100:00000001:0.0:1788573025.920908:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573025.920914:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573025.920919:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573025.920926:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573025.920932:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573025.920938:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573025.920945:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573025.920950:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573025.920956:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573026.944994:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573026.945002:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573026.945009:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573026.945039:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573026.945046:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573026.945052:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.945056:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573026.945060:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573026.945064:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573027.649053:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573027.649057:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573027.649060:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573027.968943:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573027.968954:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573027.968959:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573027.968973:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573027.968987:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573027.968993:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573027.969001:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573027.969005:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573027.969011:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573028.992891:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573028.992898:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573028.992902:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573028.992909:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573028.992915:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573028.992921:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573028.992927:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573028.992948:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573028.992955:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573029.248894:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308389101, at 4308389120 00000100:00000010:0.0:1788573029.248911:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674f00. 00000020:00000001:0.0:1788573029.697037:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573029.697040:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573029.697043:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573030.016927:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573030.016938:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573030.016942:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573030.016949:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573030.016955:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573030.016962:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573030.016970:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573030.016973:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573030.016977:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573030.528919:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573030.528932:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573030.528936:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573030.528940:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573030.528945:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573030.528955:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13717 00000400:00000200:0.0:1788573030.528965:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573030.528972:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573030.528976:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573030.528979:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.528983:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573030.528986:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.528988:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573030.528993:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573030.528995:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573030.528998:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573030.529001:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573030.529020:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573030.529023:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573030.529026:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529029:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573030.529031:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529034:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573030.529037:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573030.529041:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529044:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573030.529046:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573030.529050:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573030.529053:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.529055:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573030.529057:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573030.529063:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573030.529067:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573030.529070:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573030.529072:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.529075:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573030.529078:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573030.529082:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573030.529089:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573030.529096:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573030.529104:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516520832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573030.529116:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573030.529120:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573030.529126:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516520832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573030.529148:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573030.529151:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573030.529160:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573030.529162:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573030.529165:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573030.529168:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573030.529171:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13717 00000400:00000200:0.0:1788573030.529178:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573030.529184:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573030.529188:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573030.529195:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529199:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573030.529204:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529209:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573030.529214:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573030.529217:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573030.529221:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573030.529226:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573030.529230:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573030.529236:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573030.529242:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529246:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573030.529253:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529257:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573030.529264:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573030.529269:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529283:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573030.529294:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573030.529303:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573030.529308:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.529311:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573030.529316:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573030.529327:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.529359:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573030.529367:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573030.529373:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573030.529380:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573030.529383:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.529386:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573030.529390:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573030.529394:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573030.529401:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788573030.529409:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573030.529420:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516520960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573030.529439:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573030.529444:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573030.529452:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516520960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573030.529469:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573030.529472:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573030.529478:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573030.529483:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573030.529488:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573030.529494:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573030.529499:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13717 00000400:00000200:0.0:1788573030.529507:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573030.529516:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573030.529520:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573030.529528:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529534:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573030.529540:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529552:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573030.529557:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573030.529561:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573030.529566:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573030.529571:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573030.529576:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573030.529581:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573030.529590:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529594:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573030.529597:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529599:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573030.529604:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573030.529611:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529617:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573030.529623:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573030.529627:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573030.529632:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.529637:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573030.529641:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573030.529646:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.529650:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573030.529658:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573030.529662:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573030.529667:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573030.529669:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.529674:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573030.529677:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573030.529682:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573030.529689:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573030.529696:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573030.529706:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516521088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573030.529753:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573030.529757:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573030.529769:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516521088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573030.529786:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573030.529791:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573030.529798:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573030.529802:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573030.529805:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573030.529839:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573030.529845:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13717 00000400:00000200:0.0:1788573030.529855:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573030.529864:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573030.529870:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573030.529878:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529883:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573030.529890:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529896:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573030.529903:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573030.529907:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573030.529911:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573030.529915:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573030.529921:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573030.529928:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573030.529934:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529940:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573030.529947:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529951:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573030.529964:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573030.529971:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573030.529978:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573030.529983:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573030.529988:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573030.529994:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.530000:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573030.530005:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573030.530009:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.530013:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573030.530019:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573030.530024:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573030.530031:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573030.530034:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.530040:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573030.530042:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573030.530047:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573030.530055:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573030.530061:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573030.530072:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516521216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573030.530086:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573030.530090:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573030.530099:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516521216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573030.530113:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573030.530118:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573030.530126:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573030.530128:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573030.530132:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573030.530138:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573030.530152:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13717 00000400:00000200:0.0:1788573030.530179:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573030.530185:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573030.530190:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573030.530196:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.530203:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573030.530208:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.530212:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573030.530217:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573030.530220:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573030.530223:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573030.530227:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573030.530230:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573030.530235:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573030.530241:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.530246:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573030.530250:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573030.530254:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573030.530260:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573030.530264:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573030.530268:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573030.530273:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573030.530277:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573030.530283:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.530289:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573030.530293:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573030.530298:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.530303:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573030.530310:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573030.530317:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573030.530320:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573030.530342:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573030.530347:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573030.530351:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573030.530356:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573030.530363:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573030.530371:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573030.530383:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516521344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573030.530414:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573030.530418:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573030.530426:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516521344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573030.530443:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573030.530448:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13728) 00000100:00000001:0.0:1788573031.040886:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573031.040895:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573031.040899:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573031.040907:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573031.040912:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573031.040917:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573031.040935:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573031.040939:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573031.040945:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573032.064937:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573032.064943:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573032.064947:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573032.064954:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573032.064959:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573032.064963:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573032.064982:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573032.064985:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573032.064988:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573032.769031:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573032.769035:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573032.769039:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573033.088914:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573033.088937:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573033.088941:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573033.088947:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573033.088952:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573033.088956:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573033.088968:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573033.088971:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573033.088974:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573034.112911:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573034.112921:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573034.112926:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573034.112935:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573034.112945:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573034.112951:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573034.112968:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573034.112971:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573034.112974:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573034.368924:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308394221, at 4308394240 00000100:00000010:0.0:1788573034.368944:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916741e0. 00000020:00000001:0.0:1788573034.817032:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573034.817035:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573034.817038:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573035.136921:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573035.136929:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573035.136933:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573035.136942:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573035.136950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573035.136957:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573035.136973:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573035.136977:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573035.136981:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573035.648921:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573035.648932:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573035.648936:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573035.648940:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573035.648945:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573035.648969:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13723 00000400:00000200:0.0:1788573035.648980:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573035.648989:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573035.648993:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573035.648999:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649001:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573035.649004:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649008:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573035.649012:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573035.649015:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573035.649017:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573035.649021:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573035.649024:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573035.649025:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573035.649028:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649030:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573035.649031:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649033:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573035.649035:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573035.649038:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649039:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573035.649042:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573035.649045:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573035.649048:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.649050:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573035.649051:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573035.649056:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573035.649059:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573035.649062:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573035.649063:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.649065:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573035.649067:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573035.649075:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573035.649083:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573035.649089:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573035.649096:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516521472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573035.649105:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573035.649108:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573035.649112:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516521472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573035.649135:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573035.649141:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573035.649147:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573035.649149:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573035.649154:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573035.649159:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573035.649162:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13723 00000400:00000200:0.0:1788573035.649166:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573035.649170:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573035.649172:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573035.649184:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649196:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573035.649202:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649208:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573035.649217:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573035.649220:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573035.649227:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573035.649233:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573035.649236:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573035.649254:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573035.649260:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649273:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573035.649277:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649285:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573035.649287:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573035.649289:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649291:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573035.649292:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573035.649297:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573035.649303:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.649312:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573035.649318:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573035.649322:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.649332:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573035.649337:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573035.649340:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573035.649343:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573035.649348:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.649350:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573035.649353:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573035.649357:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573035.649362:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573035.649366:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573035.649377:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516521600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573035.649404:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573035.649416:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573035.649426:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516521600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573035.649442:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573035.649450:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573035.649457:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573035.649469:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573035.649473:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573035.649479:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573035.649489:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13723 00000400:00000200:0.0:1788573035.649503:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573035.649513:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573035.649518:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573035.649525:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649533:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573035.649540:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649550:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573035.649560:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573035.649566:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573035.649570:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573035.649576:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573035.649584:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573035.649587:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573035.649590:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649592:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573035.649596:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649601:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573035.649606:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573035.649614:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649621:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573035.649624:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573035.649630:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573035.649632:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.649634:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573035.649640:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573035.649650:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.649660:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573035.649664:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573035.649668:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573035.649677:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573035.649680:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.649686:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573035.649688:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573035.649690:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573035.649693:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573035.649696:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573035.649703:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516521728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573035.649718:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573035.649720:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573035.649724:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516521728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573035.649730:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573035.649732:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573035.649735:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573035.649736:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573035.649737:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573035.649740:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573035.649748:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13723 00000400:00000200:0.0:1788573035.649761:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573035.649774:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573035.649777:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573035.649782:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649783:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573035.649785:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649790:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573035.649793:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573035.649801:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573035.649806:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573035.649848:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573035.649851:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573035.649854:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573035.649865:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649876:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573035.649889:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649899:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573035.649905:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573035.649914:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573035.649919:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573035.649924:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573035.649929:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573035.649931:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.649935:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573035.649941:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573035.649952:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.649961:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573035.649967:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573035.649973:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573035.649982:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573035.649991:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.649998:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573035.650006:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573035.650012:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573035.650030:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788573035.650042:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573035.650053:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516521856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573035.650075:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573035.650085:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573035.650111:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516521856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573035.650159:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573035.650169:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573035.650185:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573035.650194:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573035.650204:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573035.650218:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573035.650231:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13723 00000400:00000200:0.0:1788573035.650257:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573035.650267:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573035.650274:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573035.650284:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.650296:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573035.650302:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.650308:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573035.650320:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573035.650324:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573035.650329:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573035.650333:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573035.650339:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573035.650346:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573035.650355:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.650358:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573035.650364:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573035.650369:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573035.650376:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573035.650382:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573035.650389:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573035.650406:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573035.650413:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573035.650424:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.650430:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573035.650436:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573035.650444:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.650449:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573035.650457:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788573035.650466:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573035.650474:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573035.650480:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573035.650485:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573035.650491:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573035.650497:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573035.650505:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573035.650519:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573035.650532:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516521984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573035.650561:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573035.650567:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573035.650578:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516521984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573035.650607:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573035.650620:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13733) 00000100:00000001:0.0:1788573036.160934:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573036.160947:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573036.160954:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573036.160976:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573036.160986:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573036.160995:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573036.161017:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573036.161025:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573036.161031:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573037.184924:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573037.184946:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573037.184949:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573037.184954:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573037.184958:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573037.184961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573037.184978:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573037.184982:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573037.184986:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573037.185009:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573037.185012:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573037.185014:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573037.185015:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573037.185018:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573037.185021:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573037.185023:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573037.185024:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573037.185026:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573037.889016:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573037.889020:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573037.889022:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573038.208918:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573038.208926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573038.208931:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573038.208938:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573038.208947:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573038.208953:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573038.208959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573038.208962:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573038.208966:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573039.232903:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573039.232909:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573039.232913:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573039.232919:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573039.232924:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573039.232929:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573039.232936:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573039.232941:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573039.232950:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573039.488890:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308399341, at 4308399360 00000100:00000010:0.0:1788573039.488907:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c60. 00000020:00000001:0.0:1788573039.937026:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573039.937029:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573039.937032:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573040.256933:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573040.256939:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573040.256941:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573040.256949:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573040.256954:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573040.256958:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573040.256962:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573040.256967:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573040.256972:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573040.256987:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573040.256992:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573040.256994:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573040.256997:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573040.257000:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573040.257002:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573040.257004:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573040.257005:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573040.257008:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.280909:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573041.280917:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573041.280920:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573041.280929:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573041.280935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573041.280941:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573041.280947:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573041.280949:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573041.280952:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573041.280976:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573041.280980:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573041.280983:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573041.280987:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573041.280992:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573041.281003:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573041.281006:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573041.281007:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573041.281009:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573042.304934:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573042.304949:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573042.304954:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573042.304961:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573042.304965:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573042.304972:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573042.304982:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573042.304988:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573042.304994:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573043.008981:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573043.008985:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573043.008988:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573043.328928:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573043.328934:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573043.328937:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573043.328942:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573043.328949:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573043.328957:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573043.328963:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573043.328968:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573043.328973:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573044.352907:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573044.352917:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573044.352922:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573044.352929:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573044.352934:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573044.352940:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573044.352954:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573044.352958:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573044.352964:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573044.608916:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308404462, at 4308404480 00000100:00000010:0.0:1788573044.608935:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7420. 00000020:00000001:0.0:1788573045.056994:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573045.057012:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573045.057014:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573045.376915:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573045.376925:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573045.376929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573045.376938:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573045.376945:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573045.376950:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573045.376968:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573045.376972:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573045.376976:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573046.400904:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573046.400910:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573046.400914:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573046.400919:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573046.400924:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573046.400928:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573046.400931:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573046.400935:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573046.400941:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573047.424929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573047.424940:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573047.424945:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573047.424964:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573047.424971:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573047.424975:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573047.424986:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573047.424990:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573047.424993:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573048.129054:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573048.129057:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573048.129060:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573048.448962:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573048.448967:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573048.448969:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573048.448975:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573048.448980:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573048.448984:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573048.448994:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573048.448997:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573048.449000:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573049.472940:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573049.472946:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573049.472950:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573049.472959:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573049.472965:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573049.472971:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573049.472996:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573049.473000:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573049.473005:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573049.728940:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308409581, at 4308409600 00000100:00000010:0.0:1788573049.728961:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7360. 00000020:00000001:0.0:1788573050.177065:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573050.177069:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573050.177072:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573050.496915:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573050.496931:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573050.496937:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573050.496945:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573050.496949:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573050.496952:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573050.497007:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573050.497011:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573050.497014:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573051.520919:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573051.520928:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573051.520933:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573051.520942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573051.520962:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573051.520966:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573051.520982:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573051.520986:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573051.520989:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573052.544910:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573052.544919:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573052.544951:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573052.544955:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573052.544960:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573052.544965:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573052.544993:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573052.544995:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573052.544999:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573053.249004:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573053.249008:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573053.249011:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573053.568928:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573053.568937:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573053.568941:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573053.568948:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573053.568959:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573053.568964:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573053.568973:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573053.568977:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573053.568981:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573054.592934:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573054.592944:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573054.592949:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573054.592958:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573054.592966:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573054.592972:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573054.593006:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573054.593009:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573054.593012:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573054.848898:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308414701, at 4308414720 00000100:00000010:0.0:1788573054.848909:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea73c0. 00000020:00000001:0.0:1788573055.297013:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573055.297017:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573055.297020:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573055.616947:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573055.616952:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573055.616955:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573055.616958:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573055.616971:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573055.616976:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573055.616992:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573055.616995:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573055.616998:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573056.640923:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573056.640928:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573056.640933:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573056.640938:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573056.640945:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573056.640948:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573056.640962:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573056.640965:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573056.640969:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573057.664941:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573057.664948:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573057.664951:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573057.664959:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573057.664965:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573057.664970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573057.664976:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573057.664979:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573057.664983:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573057.665011:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573057.665014:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573057.665016:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573057.665018:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573057.665020:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573057.665024:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573057.665027:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573057.665029:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573057.665033:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573058.369027:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573058.369031:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573058.369033:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573058.688903:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573058.688912:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573058.688916:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573058.688943:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573058.688950:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573058.688954:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.688969:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573058.688973:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573058.688978:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573059.712938:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573059.712945:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573059.712948:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573059.712961:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573059.712972:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573059.712979:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573059.712984:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573059.712988:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573059.712996:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573059.713036:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573059.713047:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573059.713052:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573059.713058:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573059.713063:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573059.713066:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573059.713068:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573059.713069:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573059.713071:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573059.968927:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308419821, at 4308419840 00000100:00000010:0.0:1788573059.968954:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7540. 00000020:00000001:0.0:1788573060.417028:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573060.417031:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573060.417034:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573060.736894:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573060.736904:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573060.736908:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573060.736917:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573060.736923:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573060.736928:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573060.736940:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573060.736944:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573060.736969:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573061.248927:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573061.248941:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573061.248944:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573061.248949:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573061.248955:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573061.248961:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13748 00000400:00000200:0.0:1788573061.248969:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573061.248974:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573061.248976:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573061.248978:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.248981:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573061.248982:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.248984:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573061.248988:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573061.248989:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573061.248991:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573061.248993:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573061.248995:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573061.248997:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573061.248999:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249000:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573061.249002:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249003:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573061.249005:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573061.249008:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249009:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573061.249011:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573061.249014:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573061.249016:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249031:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573061.249033:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573061.249038:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573061.249040:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573061.249043:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573061.249045:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249047:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573061.249050:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573061.249053:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573061.249059:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573061.249064:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573061.249072:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516524672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573061.249080:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573061.249083:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573061.249087:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516524672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573061.249106:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573061.249108:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573061.249111:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573061.249112:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573061.249113:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573061.249115:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573061.249117:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13748 00000400:00000200:0.0:1788573061.249122:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573061.249125:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573061.249128:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573061.249131:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249133:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573061.249135:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249138:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573061.249147:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573061.249149:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573061.249150:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573061.249153:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573061.249156:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573061.249159:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573061.249165:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249169:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573061.249173:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249176:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573061.249179:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573061.249183:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249187:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573061.249191:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573061.249198:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573061.249201:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249204:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573061.249208:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573061.249211:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249214:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573061.249217:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573061.249221:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573061.249227:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573061.249228:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249231:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573061.249232:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573061.249235:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573061.249239:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573061.249244:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573061.249251:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516524800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573061.249265:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573061.249275:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573061.249284:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516524800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573061.249300:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573061.249305:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573061.249309:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573061.249312:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573061.249315:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573061.249318:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573061.249322:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13748 00000400:00000200:0.0:1788573061.249328:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573061.249334:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573061.249338:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573061.249343:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249347:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573061.249352:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249356:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573061.249360:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573061.249362:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573061.249364:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573061.249367:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573061.249369:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573061.249373:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573061.249376:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249381:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573061.249383:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249385:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573061.249391:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573061.249393:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249401:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573061.249404:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573061.249407:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573061.249411:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249414:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573061.249418:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573061.249420:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249421:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573061.249427:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573061.249429:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573061.249431:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573061.249432:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249433:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573061.249436:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573061.249440:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573061.249444:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573061.249450:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573061.249458:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516524928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573061.249471:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573061.249475:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573061.249481:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516524928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573061.249487:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573061.249489:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573061.249491:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573061.249494:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573061.249496:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573061.249500:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573061.249504:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13748 00000400:00000200:0.0:1788573061.249511:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573061.249518:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573061.249520:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573061.249523:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249528:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573061.249533:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249536:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573061.249541:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573061.249542:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573061.249543:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573061.249544:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573061.249545:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573061.249547:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573061.249550:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249552:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573061.249556:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249558:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573061.249563:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573061.249564:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249567:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573061.249572:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573061.249574:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573061.249577:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249580:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573061.249582:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573061.249585:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249587:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573061.249592:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573061.249595:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573061.249598:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573061.249600:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249605:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573061.249610:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573061.249613:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573061.249618:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573061.249624:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573061.249629:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516525056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573061.249636:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573061.249640:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573061.249645:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516525056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573061.249659:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573061.249661:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573061.249666:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573061.249669:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573061.249672:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573061.249674:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573061.249677:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13748 00000400:00000200:0.0:1788573061.249682:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573061.249686:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573061.249690:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573061.249695:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249699:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573061.249703:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249707:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573061.249710:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573061.249712:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573061.249714:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573061.249718:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573061.249721:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573061.249724:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573061.249735:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249739:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573061.249743:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249745:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573061.249751:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573061.249754:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573061.249757:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573061.249758:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573061.249760:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573061.249761:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249762:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573061.249764:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573061.249779:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249782:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573061.249787:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573061.249790:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573061.249793:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573061.249796:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.249797:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573061.249800:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573061.249803:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573061.249826:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573061.249829:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573061.249834:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516525184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573061.249843:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573061.249844:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573061.249848:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516525184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573061.249854:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573061.249862:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13758) 00000100:00000001:0.0:1788573061.760972:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573061.760981:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573061.760991:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573061.761006:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573061.761024:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573061.761035:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573061.761047:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573061.761052:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573061.761054:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573061.761069:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573061.761072:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573061.761083:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573061.761102:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573061.761110:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573061.761120:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573061.761128:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573061.761133:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573061.761137:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573062.785043:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573062.785053:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573062.785060:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573062.785068:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573062.785077:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573062.785083:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573062.785098:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573062.785103:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573062.785107:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573063.489023:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573063.489027:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573063.489029:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573063.808915:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573063.808923:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573063.808927:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573063.808934:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573063.808942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573063.808947:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573063.808958:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573063.808960:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573063.808973:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573064.832935:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573064.832946:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573064.832951:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573064.832970:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573064.832981:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573064.832992:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.833028:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573064.833031:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573064.833035:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573065.088882:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308424941, at 4308424960 00000100:00000010:0.0:1788573065.088898:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea79c0. 00000020:00000001:0.0:1788573065.537099:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573065.537105:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573065.537109:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573065.856998:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573065.857014:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573065.857028:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573065.857051:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573065.857063:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573065.857071:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573065.857092:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573065.857102:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573065.857107:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573066.368937:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573066.368949:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573066.368951:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573066.368955:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573066.368961:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573066.368970:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13753 00000400:00000200:0.0:1788573066.368981:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573066.368988:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573066.368990:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573066.368993:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369014:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573066.369018:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369020:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573066.369024:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573066.369026:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573066.369028:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573066.369030:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573066.369032:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573066.369035:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573066.369039:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369041:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573066.369044:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369046:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573066.369048:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573066.369051:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369053:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573066.369055:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573066.369057:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573066.369059:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.369061:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573066.369062:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573066.369067:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788573066.369070:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573066.369073:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573066.369074:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.369077:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573066.369078:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573066.369082:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573066.369088:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573066.369094:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573066.369101:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516525312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573066.369117:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573066.369121:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573066.369124:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516525312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573066.369144:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573066.369147:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573066.369149:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573066.369151:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573066.369152:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573066.369154:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573066.369156:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13753 00000400:00000200:0.0:1788573066.369160:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573066.369163:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573066.369165:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573066.369167:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369169:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573066.369171:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369175:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573066.369177:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573066.369180:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573066.369183:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573066.369185:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573066.369188:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573066.369190:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573066.369196:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369207:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573066.369211:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369215:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573066.369223:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573066.369227:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369231:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573066.369233:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573066.369240:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573066.369243:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.369246:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573066.369251:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573066.369261:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.369264:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573066.369270:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573066.369274:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573066.369275:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573066.369276:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.369278:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573066.369279:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573066.369281:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573066.369287:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573066.369291:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573066.369298:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516525440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573066.369317:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573066.369322:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573066.369333:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516525440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573066.369359:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573066.369366:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573066.369373:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573066.369377:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573066.369380:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573066.369385:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573066.369396:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13753 00000400:00000200:0.0:1788573066.369405:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573066.369410:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573066.369412:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573066.369416:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369420:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573066.369425:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369430:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573066.369434:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573066.369436:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573066.369439:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573066.369441:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573066.369443:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573066.369446:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573066.369452:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369453:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573066.369455:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369457:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573066.369459:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573066.369460:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369462:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573066.369463:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573066.369467:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573066.369469:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.369471:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573066.369475:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573066.369480:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.369481:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573066.369488:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573066.369491:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573066.369494:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573066.369504:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.369515:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573066.369518:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573066.369520:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573066.369526:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573066.369531:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573066.369539:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516525568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573066.369549:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573066.369553:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573066.369561:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516525568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573066.369574:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573066.369578:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573066.369580:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573066.369583:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573066.369586:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573066.369590:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573066.369596:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13753 00000400:00000200:0.0:1788573066.369601:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573066.369605:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573066.369606:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573066.369612:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369614:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573066.369616:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369620:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573066.369621:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573066.369630:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573066.369635:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573066.369641:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573066.369646:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573066.369658:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573066.369663:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369666:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573066.369668:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369685:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573066.369688:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573066.369691:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369694:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573066.369696:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573066.369699:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573066.369701:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.369705:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573066.369708:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573066.369713:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.369715:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573066.369719:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573066.369723:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573066.369729:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573066.369732:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.369736:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573066.369739:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573066.369742:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573066.369750:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573066.369757:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573066.369769:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516525696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573066.369786:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573066.369791:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573066.369802:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516525696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573066.369861:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573066.369873:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573066.369882:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573066.369886:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573066.369891:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573066.369897:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573066.369904:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13753 00000400:00000200:0.0:1788573066.369916:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573066.369924:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573066.369927:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573066.369930:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369936:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573066.369940:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369945:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573066.369949:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573066.369953:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573066.369956:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573066.369960:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573066.369965:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573066.369970:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573066.369977:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369981:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573066.369986:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573066.369991:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573066.369995:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573066.370003:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573066.370008:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573066.370013:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573066.370019:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573066.370025:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.370029:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573066.370041:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573066.370045:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.370048:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573066.370054:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573066.370058:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573066.370065:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573066.370068:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573066.370072:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573066.370076:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573066.370079:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573066.370088:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573066.370094:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573066.370105:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516525824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573066.370121:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573066.370126:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573066.370135:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516525824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573066.370151:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573066.370158:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13763) 00000100:00000001:0.0:1788573066.880950:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573066.880956:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573066.880959:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573066.880966:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573066.880970:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573066.880976:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573066.880996:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573066.881001:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573066.881005:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573067.904936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573067.904939:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573067.904942:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573067.904951:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573067.904960:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573067.904985:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573067.905004:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573067.905016:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573067.905025:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573068.609062:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573068.609068:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573068.609073:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573068.928879:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573068.928888:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573068.928890:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573068.928906:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573068.928914:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573068.928921:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573068.928930:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573068.928935:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573068.928940:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573069.952897:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573069.952906:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573069.952909:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573069.952918:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573069.952927:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573069.952933:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.952947:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573069.952950:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573069.952956:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573070.208927:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308430061, at 4308430080 00000100:00000010:0.0:1788573070.208943:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f60. 00000020:00000001:0.0:1788573070.657058:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573070.657064:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573070.657069:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573070.976991:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573070.976999:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573070.977011:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573070.977017:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573070.977026:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573070.977038:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.977044:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573070.977054:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573070.977057:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573071.488893:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573071.488904:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573071.488907:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573071.488913:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573071.488918:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573071.488927:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13758 00000400:00000200:0.0:1788573071.488937:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573071.488945:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573071.488949:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573071.488952:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.488955:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573071.488958:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.488960:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573071.488965:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573071.488967:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573071.488970:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573071.488974:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573071.488976:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573071.488979:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573071.488982:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.488984:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573071.488986:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.488988:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573071.488990:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573071.488992:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573071.488994:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573071.488997:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573071.489000:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573071.489003:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489023:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573071.489025:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573071.489030:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573071.489033:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573071.489036:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573071.489037:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489040:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573071.489042:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573071.489046:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573071.489051:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573071.489057:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573071.489065:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516525952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573071.489073:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573071.489076:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573071.489079:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516525952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573071.489100:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573071.489103:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573071.489106:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573071.489108:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573071.489109:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573071.489111:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573071.489114:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13758 00000400:00000200:0.0:1788573071.489118:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573071.489121:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573071.489123:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573071.489126:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489130:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573071.489132:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489136:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573071.489146:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573071.489149:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573071.489152:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573071.489155:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573071.489160:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573071.489166:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573071.489173:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489177:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573071.489181:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489184:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573071.489188:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573071.489192:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489196:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573071.489198:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573071.489210:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573071.489215:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489218:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573071.489223:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573071.489226:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489230:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573071.489234:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573071.489237:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573071.489240:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573071.489242:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489244:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573071.489247:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573071.489250:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573071.489257:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573071.489261:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573071.489269:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516526080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573071.489278:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573071.489287:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573071.489299:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516526080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573071.489313:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573071.489316:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573071.489321:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573071.489324:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573071.489325:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573071.489329:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573071.489335:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13758 00000400:00000200:0.0:1788573071.489340:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573071.489347:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573071.489350:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573071.489356:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489361:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573071.489366:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489370:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573071.489374:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573071.489378:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573071.489380:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573071.489382:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573071.489385:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573071.489390:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573071.489395:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489399:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573071.489403:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489406:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573071.489408:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573071.489412:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489422:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573071.489424:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573071.489428:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573071.489433:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489436:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573071.489439:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573071.489444:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489448:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573071.489452:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573071.489456:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573071.489461:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573071.489463:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489466:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573071.489468:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573071.489470:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573071.489478:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573071.489483:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573071.489492:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516526208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573071.489500:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573071.489503:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573071.489510:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516526208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573071.489524:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573071.489526:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573071.489533:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573071.489536:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573071.489553:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573071.489555:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573071.489557:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13758 00000400:00000200:0.0:1788573071.489560:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573071.489572:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573071.489575:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573071.489578:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489583:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573071.489586:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489589:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573071.489592:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573071.489595:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573071.489597:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573071.489598:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573071.489599:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573071.489601:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573071.489602:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489605:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573071.489608:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489610:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573071.489612:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573071.489614:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489615:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573071.489617:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573071.489621:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573071.489629:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489632:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573071.489637:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573071.489639:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489645:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573071.489652:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573071.489655:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573071.489659:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573071.489661:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489665:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573071.489670:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573071.489672:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573071.489678:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573071.489690:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573071.489698:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516526336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573071.489724:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573071.489725:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573071.489731:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516526336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573071.489755:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573071.489758:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573071.489763:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573071.489765:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573071.489769:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573071.489774:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573071.489779:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13758 00000400:00000200:0.0:1788573071.489782:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573071.489784:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573071.489788:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573071.489793:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489797:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573071.489801:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489804:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573071.489833:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573071.489835:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573071.489838:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573071.489841:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573071.489845:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573071.489848:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573071.489861:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489866:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573071.489870:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489873:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573071.489878:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573071.489882:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573071.489886:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573071.489888:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573071.489892:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573071.489897:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489900:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573071.489903:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573071.489908:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489911:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573071.489917:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573071.489920:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=5 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573071.489926:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573071.489928:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573071.489932:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573071.489935:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573071.489939:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573071.489943:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788573071.489949:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573071.489956:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516526464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573071.489965:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573071.489970:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573071.489975:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516526464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573071.489982:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573071.489985:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13769) 00000100:00000001:0.0:1788573072.000940:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573072.000947:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573072.000955:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573072.000963:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573072.000967:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573072.000978:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573072.000997:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573072.001001:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573072.001004:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573073.024952:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573073.024960:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573073.024970:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573073.024979:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573073.024994:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573073.024999:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573073.025021:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573073.025026:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573073.025029:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788573073.537265:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573073.537283:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573073.537290:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573073.537293:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.729184:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573073.729189:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573073.729197:0:241699:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 36 locks of 0 total in blp. (0 blwis in pool) 00010000:00000001:0.0:1788573073.729203:0:241699:0:(ldlm_lockd.c:2881:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1788573073.729206:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.729213:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000045df0a51/0x7d52aadf1286cd39 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5af9e expref: -99 pid: 260122 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.729231:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.729235:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.729240:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.729244:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.729247:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.729252:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.729254:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.729266:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.729271:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.729277:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.729281:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.729286:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.729289:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.729291:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.729293:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8731c00 -> ffff8fc4acb5a148 00000020:00000001:0.0:1788573073.729296:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acb5a148 -> ffff8fc4acb5a148 00000020:00000001:0.0:1788573073.729298:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.729304:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.729306:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1052:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.729311:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.729314:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.729317:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.729324:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1052:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.729329:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.729332:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.729335:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.729337:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.729341:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.729343:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.729345:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.729348:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.729350:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.729356:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.729360:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.729363:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.729367:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.729369:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.729372:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.729375:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1052:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.729380:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.729382:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.729386:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.729388:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.729392:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.729395:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8731c00 -> ffff8fc4acb5a148 00010000:00000001:0.0:1788573073.729405:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.729411:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.729416:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8731c00 -> ffff8fc4acb5a148 00000020:00000001:0.0:1788573073.729419:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.729422:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.729426:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.729429:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.729432:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.729435:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8731c00 -> ffff8fc4acb5a148 00000020:00000001:0.0:1788573073.729438:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acb5a148 -> ffff8fc4acb5a148 00000020:00000001:0.0:1788573073.729441:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.729509:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.729516:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8731c00 -> ffff8fc4acb5a148 00000008:00000001:0.0:1788573073.729523:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.729526:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.729532:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.729539:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.729546:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.729550:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.729555:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.729561:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.729570:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a635da00 with handle 0x7d52aadf1286cd39 from hash 00010000:00000001:0.0:1788573073.729576:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.729579:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.729583:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.729587:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.729591:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.729595:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.729602:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.729609:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000004c6fe2f8/0x7d52aadf1286cd1d lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5af58 expref: -99 pid: 260118 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.729632:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.729635:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.729639:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.729644:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.729654:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.729657:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.729661:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.729665:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.729669:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.729673:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.729676:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.729680:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.729682:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.729683:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.729685:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c43dc0 -> ffff8fc4accf83a0 00000020:00000001:0.0:1788573073.729686:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4accf83a0 -> ffff8fc4accf83a0 00000020:00000001:0.0:1788573073.729690:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.729693:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.729697:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1050:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.729700:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.729702:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.729705:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.729708:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1050:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.729716:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.729719:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.729722:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.729727:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.729730:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.729733:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.729736:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.729739:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.729743:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.729746:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.729750:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.729753:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.729756:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.729759:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.729762:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.729765:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1050:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.729769:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.729772:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.729775:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.729785:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.729790:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.729794:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c43dc0 -> ffff8fc4accf83a0 00010000:00000001:0.0:1788573073.729797:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.729801:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.729806:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c43dc0 -> ffff8fc4accf83a0 00000020:00000001:0.0:1788573073.729856:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.729858:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.729861:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.729865:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.729869:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.729874:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c43dc0 -> ffff8fc4accf83a0 00000020:00000001:0.0:1788573073.729879:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4accf83a0 -> ffff8fc4accf83a0 00000020:00000001:0.0:1788573073.729883:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.729886:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.729891:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c43dc0 -> ffff8fc4accf83a0 00000008:00000001:0.0:1788573073.729897:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.729902:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.729906:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.729912:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.729916:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.729920:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.729925:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.729929:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.729933:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b8f52200 with handle 0x7d52aadf1286cd1d from hash 00010000:00000001:0.0:1788573073.729937:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.729940:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.729943:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.729946:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.729950:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.729953:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.729956:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.729960:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000005ecb62b7/0x7d52aadf1286cd01 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5af12 expref: -99 pid: 260114 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.729972:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.729981:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.729985:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.729990:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.729995:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.729999:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.730002:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.730005:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.730009:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.730013:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.730016:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.730022:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.730027:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.730030:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.730033:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c42a80 -> ffff8fc4acdc6148 00000020:00000001:0.0:1788573073.730038:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acdc6148 -> ffff8fc4acdc6148 00000020:00000001:0.0:1788573073.730044:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.730047:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.730051:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x104e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.730056:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.730058:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.730059:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.730061:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x104e:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.730063:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.730064:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.730066:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.730073:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.730076:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730080:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.730084:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.730089:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.730093:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.730098:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.730103:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.730106:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.730108:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.730109:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.730109:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.730112:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x104e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.730125:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.730128:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.730131:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.730134:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.730140:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730145:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c42a80 -> ffff8fc4acdc6148 00010000:00000001:0.0:1788573073.730151:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.730156:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730161:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c42a80 -> ffff8fc4acdc6148 00000020:00000001:0.0:1788573073.730165:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.730168:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.730172:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.730176:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730179:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730183:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c42a80 -> ffff8fc4acdc6148 00000020:00000001:0.0:1788573073.730186:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acdc6148 -> ffff8fc4acdc6148 00000020:00000001:0.0:1788573073.730189:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.730192:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730195:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c42a80 -> ffff8fc4acdc6148 00000008:00000001:0.0:1788573073.730201:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.730204:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.730208:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.730213:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.730217:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.730219:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.730223:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.730226:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.730230:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a638d800 with handle 0x7d52aadf1286cd01 from hash 00010000:00000001:0.0:1788573073.730234:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.730237:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.730238:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.730239:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.730240:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.730244:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.730247:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.730251:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000000799443e/0x7d52aadf1286cce5 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5aecc expref: -99 pid: 260110 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.730284:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.730289:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.730294:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.730299:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.730304:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.730309:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.730311:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.730314:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.730319:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.730322:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.730326:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.730332:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.730337:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.730340:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.730344:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882fdc0 -> ffff8fc4b8875d00 00000020:00000001:0.0:1788573073.730347:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8875d00 -> ffff8fc4b8875d00 00000020:00000001:0.0:1788573073.730350:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.730353:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.730356:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x104c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.730362:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.730365:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.730366:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.730368:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x104c:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.730371:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.730374:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.730377:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.730383:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.730387:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730392:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.730396:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.730398:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.730402:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.730405:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.730406:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.730407:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.730418:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.730421:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.730424:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.730428:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x104c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.730434:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.730439:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.730440:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.730444:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.730449:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730467:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882fdc0 -> ffff8fc4b8875d00 00010000:00000001:0.0:1788573073.730471:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.730475:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730480:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882fdc0 -> ffff8fc4b8875d00 00000020:00000001:0.0:1788573073.730483:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.730486:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.730489:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.730493:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730496:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730500:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882fdc0 -> ffff8fc4b8875d00 00000020:00000001:0.0:1788573073.730503:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8875d00 -> ffff8fc4b8875d00 00000020:00000001:0.0:1788573073.730506:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.730510:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730514:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882fdc0 -> ffff8fc4b8875d00 00000008:00000001:0.0:1788573073.730521:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.730525:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.730528:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.730530:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.730532:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.730535:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.730538:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.730541:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.730545:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b8f53600 with handle 0x7d52aadf1286cce5 from hash 00010000:00000001:0.0:1788573073.730549:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.730551:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.730554:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.730556:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.730560:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.730564:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.730574:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.730578:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000a38b0691/0x7d52aadf1286ccc9 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5ae86 expref: -99 pid: 260106 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.730591:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.730594:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.730598:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.730604:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.730608:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.730611:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.730613:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.730617:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.730620:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.730623:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.730626:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.730632:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.730636:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.730638:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.730641:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c436c0 -> ffff8fc4acdc6c38 00000020:00000001:0.0:1788573073.730644:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acdc6c38 -> ffff8fc4acdc6c38 00000020:00000001:0.0:1788573073.730648:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.730650:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.730653:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x104a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.730659:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.730663:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.730667:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.730670:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x104a:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.730675:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.730678:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.730681:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.730684:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.730687:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730690:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.730694:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.730698:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.730701:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.730707:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.730708:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.730712:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.730714:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.730717:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.730720:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.730725:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x104a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.730729:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.730732:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.730735:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.730739:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.730743:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730747:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c436c0 -> ffff8fc4acdc6c38 00010000:00000001:0.0:1788573073.730750:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.730751:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730752:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c436c0 -> ffff8fc4acdc6c38 00000020:00000001:0.0:1788573073.730753:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.730754:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.730756:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.730758:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730762:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730765:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c436c0 -> ffff8fc4acdc6c38 00000020:00000001:0.0:1788573073.730766:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acdc6c38 -> ffff8fc4acdc6c38 00000020:00000001:0.0:1788573073.730769:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.730774:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.730777:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c436c0 -> ffff8fc4acdc6c38 00000008:00000001:0.0:1788573073.730783:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.730789:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.730793:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.730797:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.730801:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.730803:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.730826:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.730830:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.730835:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b1ae6a00 with handle 0x7d52aadf1286ccc9 from hash 00010000:00000001:0.0:1788573073.730838:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.730841:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.730845:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.730853:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.730857:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.730861:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.730868:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.730871:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000048388a30/0x7d52aadf1286ccad lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5ae40 expref: -99 pid: 260102 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.730884:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.730887:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.730890:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.730896:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.730900:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.730905:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.730907:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.730911:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.730917:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.730922:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.730924:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.730929:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.730934:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.730938:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.730941:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c428c0 -> ffff8fc4acdc65f8 00000020:00000001:0.0:1788573073.730944:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acdc65f8 -> ffff8fc4acdc65f8 00000020:00000001:0.0:1788573073.730949:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.730952:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.730955:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1048:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.730959:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.730961:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.730965:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.730970:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1048:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.730979:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.730983:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.730988:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.730994:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.730997:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731000:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.731008:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.731009:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.731011:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.731014:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.731019:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.731024:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.731026:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.731029:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.731033:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.731036:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1048:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.731041:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.731045:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.731048:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.731051:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.731057:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731063:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c428c0 -> ffff8fc4acdc65f8 00010000:00000001:0.0:1788573073.731068:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.731071:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731075:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c428c0 -> ffff8fc4acdc65f8 00000020:00000001:0.0:1788573073.731080:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.731083:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.731086:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.731089:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731093:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731098:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c428c0 -> ffff8fc4acdc65f8 00000020:00000001:0.0:1788573073.731102:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acdc65f8 -> ffff8fc4acdc65f8 00000020:00000001:0.0:1788573073.731106:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.731109:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731111:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a7c428c0 -> ffff8fc4acdc65f8 00000008:00000001:0.0:1788573073.731116:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.731119:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.731124:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.731128:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.731132:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.731135:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.731138:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.731142:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.731146:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a638d400 with handle 0x7d52aadf1286ccad from hash 00010000:00000001:0.0:1788573073.731153:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.731154:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.731155:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.731156:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.731158:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.731159:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.731160:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.731162:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000016ecdf39/0x7d52aadf1286cc91 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5adfa expref: -99 pid: 260098 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.731175:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.731181:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.731186:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.731190:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.731193:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.731196:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.731198:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.731201:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.731205:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.731207:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.731211:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.731216:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.731220:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.731225:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.731228:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424700 -> ffff8fc4acdc6210 00000020:00000001:0.0:1788573073.731232:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acdc6210 -> ffff8fc4acdc6210 00000020:00000001:0.0:1788573073.731238:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.731242:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.731246:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1046:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.731252:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.731255:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.731258:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.731262:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1046:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.731269:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.731273:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.731280:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.731285:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.731287:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731290:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.731293:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.731296:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.731299:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.731301:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.731302:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.731304:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.731305:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.731308:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.731310:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.731313:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1046:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.731319:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.731322:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.731325:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.731329:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.731333:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731340:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424700 -> ffff8fc4acdc6210 00010000:00000001:0.0:1788573073.731343:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.731346:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731350:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424700 -> ffff8fc4acdc6210 00000020:00000001:0.0:1788573073.731354:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.731357:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.731360:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.731363:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731367:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731370:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424700 -> ffff8fc4acdc6210 00000020:00000001:0.0:1788573073.731374:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acdc6210 -> ffff8fc4acdc6210 00000020:00000001:0.0:1788573073.731378:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.731382:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731386:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424700 -> ffff8fc4acdc6210 00000008:00000001:0.0:1788573073.731391:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.731396:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.731400:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.731406:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.731410:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.731412:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.731422:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.731425:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.731428:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b8f53400 with handle 0x7d52aadf1286cc91 from hash 00010000:00000001:0.0:1788573073.731430:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.731433:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.731437:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.731441:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.731445:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.731449:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.731538:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.731542:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000002fdc9c51/0x7d52aadf1286cc75 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5adb4 expref: -99 pid: 260094 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.731555:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.731559:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.731561:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.731565:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.731569:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.731572:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.731575:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.731577:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.731580:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.731586:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.731588:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.731592:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.731597:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.731601:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.731604:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8730700 -> ffff8fc4acb5a2d8 00000020:00000001:0.0:1788573073.731609:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acb5a2d8 -> ffff8fc4acb5a2d8 00000020:00000001:0.0:1788573073.731615:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.731617:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.731620:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1044:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.731623:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.731626:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.731629:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.731631:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1044:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.731641:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.731644:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.731648:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.731650:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.731654:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731656:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.731659:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.731662:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.731665:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.731668:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.731671:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.731675:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.731678:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.731680:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.731684:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.731688:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1044:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.731691:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.731694:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.731696:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.731701:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.731705:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731707:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8730700 -> ffff8fc4acb5a2d8 00010000:00000001:0.0:1788573073.731709:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.731710:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731711:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8730700 -> ffff8fc4acb5a2d8 00000020:00000001:0.0:1788573073.731712:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.731715:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.731719:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.731722:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731725:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731728:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8730700 -> ffff8fc4acb5a2d8 00000020:00000001:0.0:1788573073.731732:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acb5a2d8 -> ffff8fc4acb5a2d8 00000020:00000001:0.0:1788573073.731736:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.731740:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731746:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8730700 -> ffff8fc4acb5a2d8 00000008:00000001:0.0:1788573073.731750:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.731755:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.731760:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.731771:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.731775:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.731778:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.731781:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.731785:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.731789:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b8f53000 with handle 0x7d52aadf1286cc75 from hash 00010000:00000001:0.0:1788573073.731793:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.731797:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.731800:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.731804:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.731827:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.731831:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.731836:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.731841:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000004f30a754/0x7d52aadf1286cc59 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5ad6e expref: -99 pid: 260090 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.731851:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.731854:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.731858:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.731863:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.731866:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.731869:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.731872:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.731874:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.731878:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.731882:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.731885:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.731890:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.731894:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.731899:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.731902:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425dc0 -> ffff8fc4acdc6080 00000020:00000001:0.0:1788573073.731907:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acdc6080 -> ffff8fc4acdc6080 00000020:00000001:0.0:1788573073.731912:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.731916:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.731919:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1042:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.731926:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.731934:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.731939:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.731944:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1042:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.731949:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.731952:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.731956:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.731962:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.731965:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.731970:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.731974:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.731976:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.731979:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.731984:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.731987:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.731988:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.731989:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.731990:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.731992:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.731995:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1042:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.732002:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.732005:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.732007:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.732010:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.732014:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732018:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425dc0 -> ffff8fc4acdc6080 00010000:00000001:0.0:1788573073.732023:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.732027:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732031:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425dc0 -> ffff8fc4acdc6080 00000020:00000001:0.0:1788573073.732035:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.732037:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.732040:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.732043:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732047:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732052:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425dc0 -> ffff8fc4acdc6080 00000020:00000001:0.0:1788573073.732056:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acdc6080 -> ffff8fc4acdc6080 00000020:00000001:0.0:1788573073.732059:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.732062:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732066:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425dc0 -> ffff8fc4acdc6080 00000008:00000001:0.0:1788573073.732071:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.732082:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.732087:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.732091:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.732096:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.732099:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.732102:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.732106:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.732113:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a719b600 with handle 0x7d52aadf1286cc59 from hash 00010000:00000001:0.0:1788573073.732119:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.732122:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.732123:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.732124:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.732126:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.732128:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.732133:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.732137:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000095f3ad41/0x7d52aadf1286cc3d lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5ad28 expref: -99 pid: 260086 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.732152:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.732155:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.732159:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.732164:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.732168:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.732173:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.732177:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.732182:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.732186:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.732189:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.732192:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.732199:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.732203:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.732209:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.732212:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424e00 -> ffff8fc4b39a89e0 00000020:00000001:0.0:1788573073.732217:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b39a89e0 -> ffff8fc4b39a89e0 00000020:00000001:0.0:1788573073.732221:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.732230:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.732233:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1040:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.732239:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.732244:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.732248:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.732252:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1040:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.732257:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.732259:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.732260:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.732262:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.732265:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732267:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.732270:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.732271:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.732274:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.732277:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.732280:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.732284:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.732287:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.732291:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.732293:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.732297:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1040:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.732302:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.732305:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.732308:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.732312:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.732316:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732321:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424e00 -> ffff8fc4b39a89e0 00010000:00000001:0.0:1788573073.732324:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.732328:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732333:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424e00 -> ffff8fc4b39a89e0 00000020:00000001:0.0:1788573073.732338:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.732342:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.732347:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.732349:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732353:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732356:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424e00 -> ffff8fc4b39a89e0 00000020:00000001:0.0:1788573073.732360:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b39a89e0 -> ffff8fc4b39a89e0 00000020:00000001:0.0:1788573073.732363:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.732373:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732378:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424e00 -> ffff8fc4b39a89e0 00000008:00000001:0.0:1788573073.732382:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.732385:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.732387:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.732393:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.732397:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.732400:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.732404:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.732408:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.732414:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a638da00 with handle 0x7d52aadf1286cc3d from hash 00010000:00000001:0.0:1788573073.732420:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.732424:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.732427:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.732430:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.732433:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.732436:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.732440:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.732446:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000b2ae5e66/0x7d52aadf1286cc21 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5ace2 expref: -99 pid: 260082 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.732471:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.732474:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.732477:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.732480:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.732485:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.732490:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.732491:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.732494:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.732497:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.732501:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.732503:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.732506:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.732510:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.732515:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.732525:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424000 -> ffff8fc4b39a8d00 00000020:00000001:0.0:1788573073.732529:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b39a8d00 -> ffff8fc4b39a8d00 00000020:00000001:0.0:1788573073.732535:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.732537:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.732541:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x103e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.732545:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.732549:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.732550:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.732552:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x103e:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.732554:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.732554:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.732556:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.732559:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.732562:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732565:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.732568:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.732570:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.732572:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.732577:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.732580:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.732582:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.732584:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.732586:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.732588:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.732590:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x103e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.732597:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.732599:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.732602:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.732604:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.732608:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732612:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424000 -> ffff8fc4b39a8d00 00010000:00000001:0.0:1788573073.732614:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.732616:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732620:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424000 -> ffff8fc4b39a8d00 00000020:00000001:0.0:1788573073.732623:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.732626:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.732630:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.732632:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732636:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732645:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424000 -> ffff8fc4b39a8d00 00000020:00000001:0.0:1788573073.732649:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b39a8d00 -> ffff8fc4b39a8d00 00000020:00000001:0.0:1788573073.732655:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.732660:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732666:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424000 -> ffff8fc4b39a8d00 00000008:00000001:0.0:1788573073.732673:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.732680:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.732687:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.732695:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.732701:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.732706:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.732712:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.732716:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.732721:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a719aa00 with handle 0x7d52aadf1286cc21 from hash 00010000:00000001:0.0:1788573073.732728:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.732733:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.732738:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.732743:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.732749:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.732754:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.732761:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.732766:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000ed1865b8/0x7d52aadf1286cc05 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5ac9c expref: -99 pid: 260078 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.732782:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.732786:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.732790:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.732796:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.732801:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.732825:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.732828:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.732829:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.732833:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.732837:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.732839:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.732844:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.732854:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.732858:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.732862:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882ea80 -> ffff8fc4b8220aa8 00000020:00000001:0.0:1788573073.732865:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8220aa8 -> ffff8fc4b8220aa8 00000020:00000001:0.0:1788573073.732867:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.732868:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.732869:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x103c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.732875:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.732879:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.732883:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.732887:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x103c:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.732891:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.732894:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.732897:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.732901:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.732903:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732908:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.732910:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.732912:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.732915:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.732917:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.732921:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.732924:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.732926:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.732929:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.732932:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.732935:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x103c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.732939:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.732942:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.732945:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.732948:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.732952:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732956:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882ea80 -> ffff8fc4b8220aa8 00010000:00000001:0.0:1788573073.732960:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.732963:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732967:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882ea80 -> ffff8fc4b8220aa8 00000020:00000001:0.0:1788573073.732971:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.732974:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.732978:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.732988:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732992:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.732994:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882ea80 -> ffff8fc4b8220aa8 00000020:00000001:0.0:1788573073.732995:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8220aa8 -> ffff8fc4b8220aa8 00000020:00000001:0.0:1788573073.733000:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.733003:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733006:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882ea80 -> ffff8fc4b8220aa8 00000008:00000001:0.0:1788573073.733011:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.733016:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.733021:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.733024:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.733029:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.733033:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.733037:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.733040:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.733044:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b1ae7600 with handle 0x7d52aadf1286cc05 from hash 00010000:00000001:0.0:1788573073.733049:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.733054:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.733057:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.733061:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.733065:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.733069:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.733073:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.733078:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000bce5c18c/0x7d52aadf1286cbe9 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5ac56 expref: -99 pid: 260074 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.733091:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.733096:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.733100:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.733105:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.733108:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.733112:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.733114:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.733117:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.733123:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.733127:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.733135:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.733140:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.733144:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.733148:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.733152:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424380 -> ffff8fc4b39a86c0 00000020:00000001:0.0:1788573073.733157:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b39a86c0 -> ffff8fc4b39a86c0 00000020:00000001:0.0:1788573073.733161:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.733164:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.733166:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x103a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.733170:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.733173:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.733176:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.733180:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x103a:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.733188:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.733191:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.733193:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.733197:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.733200:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733203:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.733205:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.733208:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.733211:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.733215:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.733219:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.733224:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.733227:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.733230:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.733234:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.733238:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x103a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.733242:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.733245:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.733248:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.733252:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.733257:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733261:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424380 -> ffff8fc4b39a86c0 00010000:00000001:0.0:1788573073.733266:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.733269:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733273:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424380 -> ffff8fc4b39a86c0 00000020:00000001:0.0:1788573073.733283:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.733287:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.733292:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.733296:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733300:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733305:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424380 -> ffff8fc4b39a86c0 00000020:00000001:0.0:1788573073.733309:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b39a86c0 -> ffff8fc4b39a86c0 00000020:00000001:0.0:1788573073.733313:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.733316:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733321:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424380 -> ffff8fc4b39a86c0 00000008:00000001:0.0:1788573073.733327:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.733330:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.733333:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.733338:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.733342:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.733345:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.733348:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.733352:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.733359:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b1a95200 with handle 0x7d52aadf1286cbe9 from hash 00010000:00000001:0.0:1788573073.733364:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.733367:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.733370:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.733373:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.733376:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.733379:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.733385:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.733390:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000141d6b03/0x7d52aadf1286cbcd lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5ac10 expref: -99 pid: 260070 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.733402:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.733403:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.733405:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.733409:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.733413:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.733418:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.733421:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.733430:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.733435:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.733438:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.733441:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.733445:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.733448:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.733449:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.733464:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0fdc0 -> ffff8fc4b38d76c0 00000020:00000001:0.0:1788573073.733467:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b38d76c0 -> ffff8fc4b38d76c0 00000020:00000001:0.0:1788573073.733470:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.733471:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.733473:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1038:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.733477:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.733479:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.733484:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.733489:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1038:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.733491:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.733493:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.733496:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.733501:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.733503:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733507:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.733509:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.733512:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.733516:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.733521:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.733525:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.733529:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.733532:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.733534:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.733537:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.733540:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1038:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.733544:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.733547:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.733549:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.733552:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.733555:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733559:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0fdc0 -> ffff8fc4b38d76c0 00010000:00000001:0.0:1788573073.733569:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.733572:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733574:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0fdc0 -> ffff8fc4b38d76c0 00000020:00000001:0.0:1788573073.733575:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.733576:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.733581:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.733583:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733586:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733592:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0fdc0 -> ffff8fc4b38d76c0 00000020:00000001:0.0:1788573073.733593:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b38d76c0 -> ffff8fc4b38d76c0 00000020:00000001:0.0:1788573073.733597:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.733600:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733604:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0fdc0 -> ffff8fc4b38d76c0 00000008:00000001:0.0:1788573073.733609:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.733614:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.733616:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.733620:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.733623:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.733626:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.733630:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.733634:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.733639:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b1a94000 with handle 0x7d52aadf1286cbcd from hash 00010000:00000001:0.0:1788573073.733644:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.733646:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.733650:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.733652:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.733656:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.733658:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.733661:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.733666:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000002de066b1/0x7d52aadf1286cbb1 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5abca expref: -99 pid: 260065 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.733680:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.733683:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.733687:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.733692:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.733701:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.733706:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.733709:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.733713:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.733717:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.733720:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.733723:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.733729:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.733734:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.733737:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.733739:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8730fc0 -> ffff8fc4acb5a850 00000020:00000001:0.0:1788573073.733740:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acb5a850 -> ffff8fc4acb5a850 00000020:00000001:0.0:1788573073.733741:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.733742:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.733744:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1036:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.733747:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.733750:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.733752:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.733758:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1036:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.733763:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.733766:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.733769:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.733773:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.733776:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733779:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.733782:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.733785:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.733788:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.733792:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.733795:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.733798:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.733801:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.733804:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.733805:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.733825:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1036:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.733828:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.733830:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.733833:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.733842:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.733848:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733852:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8730fc0 -> ffff8fc4acb5a850 00010000:00000001:0.0:1788573073.733855:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.733859:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733864:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8730fc0 -> ffff8fc4acb5a850 00000020:00000001:0.0:1788573073.733868:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.733871:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.733875:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.733878:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733881:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733884:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8730fc0 -> ffff8fc4acb5a850 00000020:00000001:0.0:1788573073.733888:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4acb5a850 -> ffff8fc4acb5a850 00000020:00000001:0.0:1788573073.733891:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.733894:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.733896:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8730fc0 -> ffff8fc4acb5a850 00000008:00000001:0.0:1788573073.733897:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.733899:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.733903:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.733906:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.733910:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.733914:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.733918:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.733920:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.733924:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b8f52400 with handle 0x7d52aadf1286cbb1 from hash 00010000:00000001:0.0:1788573073.733930:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.733933:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.733937:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.733941:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.733944:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.733948:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.733953:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.733958:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000005acd2e4a/0x7d52aadf1286cb95 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5ab84 expref: -99 pid: 260061 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.733974:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.733983:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.733987:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.733994:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.733997:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.734001:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.734005:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.734008:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.734014:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.734017:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.734021:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.734025:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.734028:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.734030:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.734031:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0ea80 -> ffff8fc4b38d7530 00000020:00000001:0.0:1788573073.734036:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b38d7530 -> ffff8fc4b38d7530 00000020:00000001:0.0:1788573073.734040:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.734043:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.734046:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1034:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.734053:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.734055:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.734059:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.734063:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1034:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.734068:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.734070:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.734073:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.734077:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.734081:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734085:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.734089:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.734091:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.734095:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.734099:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.734103:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.734106:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.734109:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.734111:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.734113:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.734116:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1034:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.734133:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.734137:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.734140:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.734145:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.734149:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734154:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0ea80 -> ffff8fc4b38d7530 00010000:00000001:0.0:1788573073.734157:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.734161:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734164:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0ea80 -> ffff8fc4b38d7530 00000020:00000001:0.0:1788573073.734165:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.734166:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.734167:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.734172:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734177:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734183:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0ea80 -> ffff8fc4b38d7530 00000020:00000001:0.0:1788573073.734186:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b38d7530 -> ffff8fc4b38d7530 00000020:00000001:0.0:1788573073.734189:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.734193:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734196:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0ea80 -> ffff8fc4b38d7530 00000008:00000001:0.0:1788573073.734201:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.734204:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.734207:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.734211:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.734216:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.734219:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.734224:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.734229:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.734233:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b8f53800 with handle 0x7d52aadf1286cb95 from hash 00010000:00000001:0.0:1788573073.734237:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.734241:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.734244:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.734249:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.734254:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.734257:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.734262:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.734267:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000a2fd5097/0x7d52aadf1286cb79 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5ab3e expref: -99 pid: 260057 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.734295:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.734296:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.734299:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.734304:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.734308:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.734313:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.734317:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.734320:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.734325:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.734330:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.734334:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.734339:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.734342:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.734346:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.734350:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0ee00 -> ffff8fc4b38d7b70 00000020:00000001:0.0:1788573073.734353:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b38d7b70 -> ffff8fc4b38d7b70 00000020:00000001:0.0:1788573073.734356:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.734360:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.734363:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1032:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.734368:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.734371:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.734374:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.734377:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1032:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.734385:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.734387:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.734389:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.734393:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.734396:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734399:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.734402:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.734404:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.734408:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.734411:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.734415:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.734417:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.734426:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.734427:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.734429:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.734433:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1032:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.734437:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.734439:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.734443:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.734447:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.734462:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734465:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0ee00 -> ffff8fc4b38d7b70 00010000:00000001:0.0:1788573073.734468:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.734470:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734474:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0ee00 -> ffff8fc4b38d7b70 00000020:00000001:0.0:1788573073.734479:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.734483:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.734488:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.734491:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734495:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734500:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0ee00 -> ffff8fc4b38d7b70 00000020:00000001:0.0:1788573073.734505:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b38d7b70 -> ffff8fc4b38d7b70 00000020:00000001:0.0:1788573073.734509:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.734513:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734516:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0ee00 -> ffff8fc4b38d7b70 00000008:00000001:0.0:1788573073.734520:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.734525:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.734530:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.734535:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.734539:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.734542:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.734546:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.734549:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.734553:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b8f53a00 with handle 0x7d52aadf1286cb79 from hash 00010000:00000001:0.0:1788573073.734556:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.734558:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.734562:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.734566:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.734570:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.734572:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.734581:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.734585:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000005fb7b720/0x7d52aadf1286cb5d lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5aaf8 expref: -99 pid: 260053 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.734599:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.734602:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.734607:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.734613:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.734614:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.734616:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.734617:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.734618:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.734619:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.734620:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.734624:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.734627:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.734631:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.734634:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.734637:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425500 -> ffff8fc4b39a8dc8 00000020:00000001:0.0:1788573073.734643:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b39a8dc8 -> ffff8fc4b39a8dc8 00000020:00000001:0.0:1788573073.734647:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.734650:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.734654:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1030:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.734660:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.734663:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.734665:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.734671:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1030:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.734676:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.734678:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.734681:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.734685:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.734688:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734691:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.734694:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.734698:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.734701:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.734709:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.734712:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.734715:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.734717:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.734719:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.734722:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.734724:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1030:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.734728:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.734730:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.734734:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.734739:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.734742:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734748:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425500 -> ffff8fc4b39a8dc8 00010000:00000001:0.0:1788573073.734752:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.734755:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734758:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425500 -> ffff8fc4b39a8dc8 00000020:00000001:0.0:1788573073.734761:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.734763:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.734764:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.734765:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734766:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734768:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425500 -> ffff8fc4b39a8dc8 00000020:00000001:0.0:1788573073.734771:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b39a8dc8 -> ffff8fc4b39a8dc8 00000020:00000001:0.0:1788573073.734774:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.734778:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734781:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425500 -> ffff8fc4b39a8dc8 00000008:00000001:0.0:1788573073.734785:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.734788:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.734792:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.734797:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.734801:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.734804:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.734826:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.734831:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.734837:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b8f52800 with handle 0x7d52aadf1286cb5d from hash 00010000:00000001:0.0:1788573073.734843:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.734846:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.734850:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.734859:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.734862:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.734865:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.734871:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.734876:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000a7f306f4/0x7d52aadf1286cb41 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5aab2 expref: -99 pid: 260049 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.734893:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.734896:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.734900:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.734903:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.734907:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.734910:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.734911:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.734912:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.734914:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.734919:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.734924:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.734929:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.734932:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.734935:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.734938:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0f6c0 -> ffff8fc4b38d79e0 00000020:00000001:0.0:1788573073.734943:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b38d79e0 -> ffff8fc4b38d79e0 00000020:00000001:0.0:1788573073.734947:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.734951:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.734954:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x102e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.734957:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.734960:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.734964:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.734970:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x102e:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.734975:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.734978:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.734981:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.734985:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.734989:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.734994:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.735003:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.735006:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.735009:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.735012:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.735016:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.735020:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.735022:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.735025:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.735028:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.735032:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x102e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.735039:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.735043:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.735046:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.735049:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.735052:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735054:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0f6c0 -> ffff8fc4b38d79e0 00010000:00000001:0.0:1788573073.735055:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.735059:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735065:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0f6c0 -> ffff8fc4b38d79e0 00000020:00000001:0.0:1788573073.735071:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.735075:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.735079:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.735083:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735088:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735093:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0f6c0 -> ffff8fc4b38d79e0 00000020:00000001:0.0:1788573073.735097:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b38d79e0 -> ffff8fc4b38d79e0 00000020:00000001:0.0:1788573073.735100:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.735103:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735106:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0f6c0 -> ffff8fc4b38d79e0 00000008:00000001:0.0:1788573073.735111:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.735115:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.735119:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.735123:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.735128:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.735132:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.735135:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.735139:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.735144:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b8f52c00 with handle 0x7d52aadf1286cb41 from hash 00010000:00000001:0.0:1788573073.735156:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.735160:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.735164:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.735168:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.735172:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.735175:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.735179:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.735181:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000115854c3/0x7d52aadf1286cb25 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5aa6c expref: -99 pid: 260045 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.735192:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.735195:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.735198:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.735202:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.735208:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.735211:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.735212:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.735213:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.735214:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.735215:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.735216:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.735220:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.735224:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.735227:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.735231:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425a40 -> ffff8fc485e13210 00000020:00000001:0.0:1788573073.735235:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc485e13210 -> ffff8fc485e13210 00000020:00000001:0.0:1788573073.735239:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.735242:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.735245:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x102c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.735249:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.735253:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.735255:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.735260:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x102c:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.735265:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.735267:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.735276:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.735280:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.735284:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735289:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.735291:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.735294:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.735298:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.735302:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.735305:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.735310:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.735312:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.735315:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.735316:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.735320:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x102c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.735325:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.735328:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.735329:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.735330:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.735332:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735336:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425a40 -> ffff8fc485e13210 00010000:00000001:0.0:1788573073.735340:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.735344:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735347:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425a40 -> ffff8fc485e13210 00000020:00000001:0.0:1788573073.735352:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.735355:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.735358:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.735362:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735366:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735369:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425a40 -> ffff8fc485e13210 00000020:00000001:0.0:1788573073.735374:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc485e13210 -> ffff8fc485e13210 00000020:00000001:0.0:1788573073.735378:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.735383:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735387:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0425a40 -> ffff8fc485e13210 00000008:00000001:0.0:1788573073.735392:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.735395:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.735399:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.735403:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.735408:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.735412:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.735422:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.735425:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.735430:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a641a600 with handle 0x7d52aadf1286cb25 from hash 00010000:00000001:0.0:1788573073.735435:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.735439:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.735442:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.735446:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.735449:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.735450:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.735463:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.735466:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000001625ffb1/0x7d52aadf1286cb09 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5aa26 expref: -99 pid: 260041 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.735477:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.735480:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.735484:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.735487:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.735490:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.735493:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.735495:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.735497:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.735501:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.735506:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.735509:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.735515:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.735519:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.735525:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.735529:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424fc0 -> ffff8fc485e13468 00000020:00000001:0.0:1788573073.735532:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc485e13468 -> ffff8fc485e13468 00000020:00000001:0.0:1788573073.735535:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.735537:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.735539:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x102a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.735543:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.735546:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.735548:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.735552:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x102a:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.735564:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.735567:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.735571:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.735577:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.735578:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735582:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.735585:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.735587:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.735591:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.735594:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.735597:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.735601:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.735603:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.735606:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.735608:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.735611:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x102a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.735615:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.735617:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.735618:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.735622:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.735626:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735630:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424fc0 -> ffff8fc485e13468 00010000:00000001:0.0:1788573073.735633:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.735637:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735639:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424fc0 -> ffff8fc485e13468 00000020:00000001:0.0:1788573073.735641:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.735642:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.735643:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.735644:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735646:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735650:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424fc0 -> ffff8fc485e13468 00000020:00000001:0.0:1788573073.735655:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc485e13468 -> ffff8fc485e13468 00000020:00000001:0.0:1788573073.735659:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.735662:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735666:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4a0424fc0 -> ffff8fc485e13468 00000008:00000001:0.0:1788573073.735671:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.735677:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.735684:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.735694:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.735699:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.735702:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.735706:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.735709:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.735714:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a641bc00 with handle 0x7d52aadf1286cb09 from hash 00010000:00000001:0.0:1788573073.735718:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.735721:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.735724:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.735727:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.735732:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.735735:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.735739:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.735744:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000062314496/0x7d52aadf1286caed lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a9e0 expref: -99 pid: 260037 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.735757:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.735760:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.735765:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.735769:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.735773:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.735775:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.735776:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.735777:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.735778:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.735779:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.735781:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.735786:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.735789:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.735792:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.735795:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882ee00 -> ffff8fc4b82206c0 00000020:00000001:0.0:1788573073.735798:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b82206c0 -> ffff8fc4b82206c0 00000020:00000001:0.0:1788573073.735802:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.735805:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.735828:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1028:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.735834:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.735843:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.735847:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.735854:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1028:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.735861:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.735864:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.735867:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.735871:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.735874:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735877:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.735880:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.735883:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.735886:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.735891:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.735894:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.735898:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.735901:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.735903:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.735906:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.735909:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1028:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.735913:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.735915:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.735918:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.735920:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.735924:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735929:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882ee00 -> ffff8fc4b82206c0 00010000:00000001:0.0:1788573073.735931:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.735932:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735933:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882ee00 -> ffff8fc4b82206c0 00000020:00000001:0.0:1788573073.735937:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.735939:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.735944:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.735947:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735949:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735952:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882ee00 -> ffff8fc4b82206c0 00000020:00000001:0.0:1788573073.735956:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b82206c0 -> ffff8fc4b82206c0 00000020:00000001:0.0:1788573073.735960:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.735963:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.735966:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882ee00 -> ffff8fc4b82206c0 00000008:00000001:0.0:1788573073.735970:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.735980:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.735984:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.735991:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.735995:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.735998:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.736002:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.736007:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.736011:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc484dd8a00 with handle 0x7d52aadf1286caed from hash 00010000:00000001:0.0:1788573073.736015:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.736018:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.736021:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.736023:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.736027:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.736030:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.736035:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.736040:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000094e5b9b9/0x7d52aadf1286cad1 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a99a expref: -99 pid: 260033 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.736055:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.736058:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.736060:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.736062:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.736065:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.736070:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.736073:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.736076:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.736080:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.736084:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.736087:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.736091:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.736095:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.736100:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.736104:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8aa80 -> ffff8fc485e13918 00000020:00000001:0.0:1788573073.736110:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc485e13918 -> ffff8fc485e13918 00000020:00000001:0.0:1788573073.736116:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.736126:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.736129:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1026:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.736136:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.736139:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.736142:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.736146:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1026:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.736153:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.736156:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.736160:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.736163:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.736167:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736170:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.736173:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.736176:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.736179:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.736182:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.736187:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.736191:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.736192:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.736193:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.736194:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.736195:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1026:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.736201:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.736205:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.736206:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.736209:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.736213:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736217:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8aa80 -> ffff8fc485e13918 00010000:00000001:0.0:1788573073.736221:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.736223:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736227:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8aa80 -> ffff8fc485e13918 00000020:00000001:0.0:1788573073.736231:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.736234:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.736237:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.736240:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736245:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736248:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8aa80 -> ffff8fc485e13918 00000020:00000001:0.0:1788573073.736252:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc485e13918 -> ffff8fc485e13918 00000020:00000001:0.0:1788573073.736256:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.736265:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736270:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8aa80 -> ffff8fc485e13918 00000008:00000001:0.0:1788573073.736276:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.736280:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.736286:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.736292:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.736296:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.736300:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.736305:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.736310:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.736315:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a641a200 with handle 0x7d52aadf1286cad1 from hash 00010000:00000001:0.0:1788573073.736319:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.736321:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.736322:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.736323:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.736326:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.736328:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.736332:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.736337:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000075c7a7b1/0x7d52aadf1286cab5 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a954 expref: -99 pid: 260029 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.736354:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.736357:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.736361:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.736365:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.736368:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.736372:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.736375:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.736378:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.736382:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.736385:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.736388:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.736391:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.736394:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.736397:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.736407:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8bdc0 -> ffff8fc485e13c38 00000020:00000001:0.0:1788573073.736413:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc485e13c38 -> ffff8fc485e13c38 00000020:00000001:0.0:1788573073.736417:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.736421:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.736425:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1024:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.736430:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.736433:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.736436:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.736439:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1024:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.736444:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.736447:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.736450:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.736462:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.736463:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736464:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.736465:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.736468:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.736470:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.736473:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.736475:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.736480:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.736483:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.736485:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.736487:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.736491:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1024:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.736494:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.736496:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.736498:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.736501:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.736505:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736509:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8bdc0 -> ffff8fc485e13c38 00010000:00000001:0.0:1788573073.736513:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.736517:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736521:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8bdc0 -> ffff8fc485e13c38 00000020:00000001:0.0:1788573073.736525:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.736527:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.736530:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.736533:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736537:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736549:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8bdc0 -> ffff8fc485e13c38 00000020:00000001:0.0:1788573073.736552:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc485e13c38 -> ffff8fc485e13c38 00000020:00000001:0.0:1788573073.736557:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.736559:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736562:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8bdc0 -> ffff8fc485e13c38 00000008:00000001:0.0:1788573073.736568:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.736571:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.736574:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.736579:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.736584:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.736588:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.736591:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.736594:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.736597:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc484dd9200 with handle 0x7d52aadf1286cab5 from hash 00010000:00000001:0.0:1788573073.736603:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.736606:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.736608:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.736611:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.736614:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.736618:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.736623:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.736627:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000007b4ef045/0x7d52aadf1286ca99 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a90e expref: -99 pid: 260025 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.736639:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.736641:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.736642:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.736643:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.736644:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.736646:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.736649:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.736653:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.736657:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.736660:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.736663:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.736667:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.736680:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.736685:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.736688:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a000 -> ffff8fc485e139e0 00000020:00000001:0.0:1788573073.736692:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc485e139e0 -> ffff8fc485e139e0 00000020:00000001:0.0:1788573073.736697:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.736701:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.736704:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1022:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.736709:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.736712:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.736715:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.736718:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1022:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.736724:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.736726:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.736730:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.736735:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.736739:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736743:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.736745:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.736748:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.736752:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.736757:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.736762:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.736765:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.736769:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.736773:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.736776:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.736780:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1022:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.736786:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.736787:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.736788:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.736790:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.736791:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736795:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a000 -> ffff8fc485e139e0 00010000:00000001:0.0:1788573073.736798:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.736801:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736806:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a000 -> ffff8fc485e139e0 00000020:00000001:0.0:1788573073.736825:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.736827:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.736828:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.736838:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736842:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736846:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a000 -> ffff8fc485e139e0 00000020:00000001:0.0:1788573073.736853:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc485e139e0 -> ffff8fc485e139e0 00000020:00000001:0.0:1788573073.736857:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.736862:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.736867:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a000 -> ffff8fc485e139e0 00000008:00000001:0.0:1788573073.736873:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.736878:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.736882:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.736888:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.736891:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.736894:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.736897:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.736901:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.736905:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a641b400 with handle 0x7d52aadf1286ca99 from hash 00010000:00000001:0.0:1788573073.736910:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.736914:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.736918:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.736921:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.736925:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.736928:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.736932:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.736937:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000057a2b159/0x7d52aadf1286ca7d lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a8c8 expref: -99 pid: 260021 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.736947:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.736950:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.736953:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.736958:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.736963:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.736967:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.736972:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.736975:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.736980:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.736982:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.736990:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.736996:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.736998:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.737002:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.737004:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0e000 -> ffff8fc4b38d75f8 00000020:00000001:0.0:1788573073.737008:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b38d75f8 -> ffff8fc4b38d75f8 00000020:00000001:0.0:1788573073.737012:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.737015:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.737019:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1020:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.737025:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.737028:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.737032:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.737038:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1020:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.737045:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.737047:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.737051:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.737054:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.737057:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737061:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.737063:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.737066:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.737069:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.737073:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.737077:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.737080:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.737087:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.737089:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.737093:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.737098:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1020:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.737104:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.737106:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.737109:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.737114:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.737119:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737125:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0e000 -> ffff8fc4b38d75f8 00010000:00000001:0.0:1788573073.737128:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.737131:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737136:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0e000 -> ffff8fc4b38d75f8 00000020:00000001:0.0:1788573073.737145:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.737148:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.737151:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.737155:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737160:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737163:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0e000 -> ffff8fc4b38d75f8 00000020:00000001:0.0:1788573073.737167:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b38d75f8 -> ffff8fc4b38d75f8 00000020:00000001:0.0:1788573073.737170:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.737173:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737177:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0e000 -> ffff8fc4b38d75f8 00000008:00000001:0.0:1788573073.737183:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.737188:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.737192:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.737197:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.737202:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.737206:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.737210:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.737214:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.737216:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a641b200 with handle 0x7d52aadf1286ca7d from hash 00010000:00000001:0.0:1788573073.737217:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.737219:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.737222:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.737224:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.737228:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.737231:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.737234:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.737240:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000060285844/0x7d52aadf1286ca61 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a882 expref: -99 pid: 260017 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.737254:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.737257:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.737260:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.737265:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.737268:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.737272:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.737275:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.737284:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.737288:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.737293:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.737298:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.737303:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.737307:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.737313:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.737318:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8ae00 -> ffff8fc485e133a0 00000020:00000001:0.0:1788573073.737323:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc485e133a0 -> ffff8fc485e133a0 00000020:00000001:0.0:1788573073.737326:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.737329:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.737333:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x101e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.737338:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.737340:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.737341:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.737343:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x101e:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.737349:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.737353:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.737355:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.737358:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.737362:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737365:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.737367:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.737370:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.737374:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.737377:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.737382:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.737387:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.737390:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.737392:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.737394:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.737397:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x101e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.737405:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.737408:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.737410:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.737413:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.737417:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737421:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8ae00 -> ffff8fc485e133a0 00010000:00000001:0.0:1788573073.737431:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.737436:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737442:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8ae00 -> ffff8fc485e133a0 00000020:00000001:0.0:1788573073.737446:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.737449:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.737462:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.737464:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737468:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737472:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8ae00 -> ffff8fc485e133a0 00000020:00000001:0.0:1788573073.737476:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc485e133a0 -> ffff8fc485e133a0 00000020:00000001:0.0:1788573073.737481:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.737485:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737489:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8ae00 -> ffff8fc485e133a0 00000008:00000001:0.0:1788573073.737491:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.737498:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.737500:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.737501:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.737503:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.737504:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.737508:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.737512:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.737518:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b1a95e00 with handle 0x7d52aadf1286ca61 from hash 00010000:00000001:0.0:1788573073.737520:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.737524:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.737526:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.737530:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.737534:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.737537:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.737544:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.737549:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000004ab82e27/0x7d52aadf1286ca45 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a83c expref: -99 pid: 260013 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.737560:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.737564:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.737566:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.737573:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.737584:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.737587:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.737591:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.737595:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.737598:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.737602:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.737605:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.737608:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.737612:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.737615:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.737619:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8bc00 -> ffff8fc4b8609850 00000020:00000001:0.0:1788573073.737622:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8609850 -> ffff8fc4b8609850 00000020:00000001:0.0:1788573073.737625:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.737627:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.737630:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x101c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.737632:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.737637:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.737640:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.737643:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x101c:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.737648:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.737652:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.737656:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.737659:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.737662:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737666:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.737668:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.737671:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.737674:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.737678:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.737680:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.737682:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.737682:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.737683:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.737684:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.737685:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x101c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.737690:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.737693:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.737696:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.737705:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.737709:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737713:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8bc00 -> ffff8fc4b8609850 00010000:00000001:0.0:1788573073.737716:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.737719:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737723:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8bc00 -> ffff8fc4b8609850 00000020:00000001:0.0:1788573073.737726:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.737730:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.737733:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.737736:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737741:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737744:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8bc00 -> ffff8fc4b8609850 00000020:00000001:0.0:1788573073.737748:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8609850 -> ffff8fc4b8609850 00000020:00000001:0.0:1788573073.737752:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.737755:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737757:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8bc00 -> ffff8fc4b8609850 00000008:00000001:0.0:1788573073.737762:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.737767:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.737772:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.737778:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.737782:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.737786:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.737790:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.737794:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.737801:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a641ae00 with handle 0x7d52aadf1286ca45 from hash 00010000:00000001:0.0:1788573073.737805:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.737825:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.737827:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.737831:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.737833:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.737837:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.737840:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.737843:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000010cbf922/0x7d52aadf1286ca29 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a7f6 expref: -99 pid: 260009 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.737853:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.737862:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.737866:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.737870:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.737873:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.737877:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.737880:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.737883:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.737888:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.737891:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.737894:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.737898:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.737904:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.737909:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.737913:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a1c0 -> ffff8fc4b8609aa8 00000020:00000001:0.0:1788573073.737916:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8609aa8 -> ffff8fc4b8609aa8 00000020:00000001:0.0:1788573073.737920:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.737924:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.737927:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x101a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.737931:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.737933:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.737936:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.737940:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x101a:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.737945:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.737948:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.737951:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.737954:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.737957:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.737961:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.737964:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.737965:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.737966:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.737967:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.737968:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.737970:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.737972:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.737974:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.737977:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.737981:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x101a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.737993:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.737996:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.737999:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.738004:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.738008:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738012:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a1c0 -> ffff8fc4b8609aa8 00010000:00000001:0.0:1788573073.738017:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.738020:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738023:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a1c0 -> ffff8fc4b8609aa8 00000020:00000001:0.0:1788573073.738027:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.738030:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.738033:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.738036:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738041:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738045:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a1c0 -> ffff8fc4b8609aa8 00000020:00000001:0.0:1788573073.738049:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8609aa8 -> ffff8fc4b8609aa8 00000020:00000001:0.0:1788573073.738051:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.738054:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738057:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a1c0 -> ffff8fc4b8609aa8 00000008:00000001:0.0:1788573073.738062:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.738067:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.738070:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.738075:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.738079:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.738082:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.738085:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.738089:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.738096:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc484dd8200 with handle 0x7d52aadf1286ca29 from hash 00010000:00000001:0.0:1788573073.738099:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.738100:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.738102:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.738105:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.738108:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.738112:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.738116:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.738122:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000067e14c26/0x7d52aadf1286ca0d lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a7b0 expref: -99 pid: 260005 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.738155:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.738158:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.738162:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.738167:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.738171:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.738176:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.738179:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.738183:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.738187:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.738192:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.738195:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.738199:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.738205:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.738208:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.738211:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882e000 -> ffff8fc4b8220b70 00000020:00000001:0.0:1788573073.738215:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8220b70 -> ffff8fc4b8220b70 00000020:00000001:0.0:1788573073.738220:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.738223:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.738224:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1018:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.738226:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.738226:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.738228:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.738231:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1018:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.738237:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.738240:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.738244:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.738250:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.738252:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738256:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.738258:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.738260:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.738265:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.738270:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.738273:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.738276:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.738285:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.738288:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.738291:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.738294:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1018:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.738299:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.738302:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.738306:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.738310:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.738314:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738319:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882e000 -> ffff8fc4b8220b70 00010000:00000001:0.0:1788573073.738322:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.738325:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738328:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882e000 -> ffff8fc4b8220b70 00000020:00000001:0.0:1788573073.738331:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.738334:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.738337:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.738341:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738348:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738353:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882e000 -> ffff8fc4b8220b70 00000020:00000001:0.0:1788573073.738357:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8220b70 -> ffff8fc4b8220b70 00000020:00000001:0.0:1788573073.738360:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.738361:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738363:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882e000 -> ffff8fc4b8220b70 00000008:00000001:0.0:1788573073.738364:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.738368:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.738373:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.738376:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.738380:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.738383:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.738387:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.738392:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.738396:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a731e400 with handle 0x7d52aadf1286ca0d from hash 00010000:00000001:0.0:1788573073.738401:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.738404:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.738407:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.738410:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.738414:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.738418:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.738429:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.738433:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000e53a778e/0x7d52aadf1286c9f1 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a76a expref: -99 pid: 260001 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.738450:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.738462:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.738467:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.738471:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.738475:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.738478:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.738480:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.738483:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.738489:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.738491:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.738496:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.738502:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.738508:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.738514:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.738517:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8afc0 -> ffff8fc4b8609f58 00000020:00000001:0.0:1788573073.738520:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8609f58 -> ffff8fc4b8609f58 00000020:00000001:0.0:1788573073.738522:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.738523:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.738524:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1016:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.738526:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.738529:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.738532:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.738535:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1016:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.738539:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.738543:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.738545:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.738548:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.738551:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738553:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.738555:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.738558:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.738560:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.738570:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.738574:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.738579:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.738581:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.738584:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.738586:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.738588:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1016:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.738592:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.738595:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.738598:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.738602:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.738606:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738609:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8afc0 -> ffff8fc4b8609f58 00010000:00000001:0.0:1788573073.738614:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.738618:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738623:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8afc0 -> ffff8fc4b8609f58 00000020:00000001:0.0:1788573073.738627:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.738628:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.738633:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.738634:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738638:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738641:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8afc0 -> ffff8fc4b8609f58 00000020:00000001:0.0:1788573073.738645:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8609f58 -> ffff8fc4b8609f58 00000020:00000001:0.0:1788573073.738647:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.738648:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738652:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8afc0 -> ffff8fc4b8609f58 00000008:00000001:0.0:1788573073.738656:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.738660:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.738664:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.738669:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.738674:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.738678:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.738681:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.738687:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.738692:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc484dd9e00 with handle 0x7d52aadf1286c9f1 from hash 00010000:00000001:0.0:1788573073.738698:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.738703:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.738705:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.738714:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.738717:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.738721:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.738726:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.738730:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000e0e82645/0x7d52aadf1286c9d5 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a724 expref: -99 pid: 259997 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.738747:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.738751:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.738756:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.738761:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.738765:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.738769:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.738772:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.738774:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.738779:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.738783:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.738786:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.738795:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.738801:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.738806:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.738826:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a540 -> ffff8fc4b8609788 00000020:00000001:0.0:1788573073.738830:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8609788 -> ffff8fc4b8609788 00000020:00000001:0.0:1788573073.738834:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.738838:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.738840:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1014:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.738844:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.738847:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.738850:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.738854:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1014:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.738857:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.738858:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.738859:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.738862:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.738865:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738868:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.738878:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.738882:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.738886:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.738890:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.738895:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.738899:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.738901:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.738903:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.738907:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.738911:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1014:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.738914:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.738916:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.738920:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.738923:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.738927:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738930:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a540 -> ffff8fc4b8609788 00010000:00000001:0.0:1788573073.738934:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.738937:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738942:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a540 -> ffff8fc4b8609788 00000020:00000001:0.0:1788573073.738946:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.738950:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.738954:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.738957:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738961:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738965:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a540 -> ffff8fc4b8609788 00000020:00000001:0.0:1788573073.738968:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8609788 -> ffff8fc4b8609788 00000020:00000001:0.0:1788573073.738972:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.738974:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.738977:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a540 -> ffff8fc4b8609788 00000008:00000001:0.0:1788573073.738981:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.738985:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.738988:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.738993:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.738996:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.738997:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.738998:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.739002:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.739009:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a6921e00 with handle 0x7d52aadf1286c9d5 from hash 00010000:00000001:0.0:1788573073.739021:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.739024:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.739027:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.739031:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.739035:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.739038:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.739042:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.739046:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000993cbdd5/0x7d52aadf1286c9b9 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a6de expref: -99 pid: 259993 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.739063:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.739067:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.739071:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.739076:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.739081:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.739086:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.739089:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.739092:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.739097:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.739101:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.739104:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.739109:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.739113:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.739117:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.739122:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882f180 -> ffff8fc4b8220210 00000020:00000001:0.0:1788573073.739127:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8220210 -> ffff8fc4b8220210 00000020:00000001:0.0:1788573073.739130:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.739131:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.739132:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1012:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.739138:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.739139:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.739143:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.739149:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1012:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.739154:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.739156:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.739166:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.739171:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.739174:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.739177:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.739181:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.739185:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.739189:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.739192:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.739196:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.739200:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.739204:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.739207:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.739210:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.739214:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1012:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.739220:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.739225:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.739230:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.739237:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.739243:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.739249:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882f180 -> ffff8fc4b8220210 00010000:00000001:0.0:1788573073.739256:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.739260:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.739265:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882f180 -> ffff8fc4b8220210 00000020:00000001:0.0:1788573073.739271:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.739277:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.739283:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.739289:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.739296:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.739304:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882f180 -> ffff8fc4b8220210 00000020:00000001:0.0:1788573073.739311:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8220210 -> ffff8fc4b8220210 00000020:00000001:0.0:1788573073.739317:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.739323:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.739330:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b882f180 -> ffff8fc4b8220210 00000008:00000001:0.0:1788573073.739337:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.739343:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.739350:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.739359:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.739365:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.739370:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.739384:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.739389:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.739396:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a731fa00 with handle 0x7d52aadf1286c9b9 from hash 00010000:00000001:0.0:1788573073.739404:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.739408:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.739413:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.739419:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.739424:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.739430:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.739437:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.739444:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000009b9e6238/0x7d52aadf1286c99d lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a698 expref: -99 pid: 259989 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.739475:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.739479:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.739484:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.739489:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.739493:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.739497:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.739499:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.739502:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.739509:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.739513:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.739515:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.739521:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.739529:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.739535:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.739540:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8b340 -> ffff8fc4b86092d8 00000020:00000001:0.0:1788573073.739546:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b86092d8 -> ffff8fc4b86092d8 00000020:00000001:0.0:1788573073.739552:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.739555:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.739559:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x1010:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.739565:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.739570:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.739576:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.739583:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x1010:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.739600:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.739603:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.739608:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.739615:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.739618:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.739622:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.739626:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.739630:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.739633:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.739638:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.739643:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.739650:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.739654:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.739658:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.739664:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.739668:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x1010:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.739676:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.739679:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.739682:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.739689:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.739695:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.739703:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8b340 -> ffff8fc4b86092d8 00010000:00000001:0.0:1788573073.739711:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.739716:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.739724:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8b340 -> ffff8fc4b86092d8 00000020:00000001:0.0:1788573073.739731:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.739736:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.739740:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.739746:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.739753:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.739762:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8b340 -> ffff8fc4b86092d8 00000020:00000001:0.0:1788573073.739767:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b86092d8 -> ffff8fc4b86092d8 00000020:00000001:0.0:1788573073.739773:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.739777:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.739784:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8b340 -> ffff8fc4b86092d8 00000008:00000001:0.0:1788573073.739791:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.739797:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.739803:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.739846:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.739852:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.739857:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.739863:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.739869:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.739877:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a731fe00 with handle 0x7d52aadf1286c99d from hash 00010000:00000001:0.0:1788573073.739885:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.739891:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.739895:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.739901:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.739907:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.739913:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.739921:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.739926:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000a45b0440/0x7d52aadf1286c981 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a652 expref: -99 pid: 259985 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.739950:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.739954:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.739960:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.739967:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.739972:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.739978:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.739983:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.739988:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.739995:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.740000:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.740006:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.740013:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.740020:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.740027:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.740032:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a380 -> ffff8fc4b8609918 00000020:00000001:0.0:1788573073.740039:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8609918 -> ffff8fc4b8609918 00000020:00000001:0.0:1788573073.740045:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.740047:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.740051:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x100e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.740059:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.740072:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.740077:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.740086:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x100e:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.740097:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.740100:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.740105:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.740113:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.740117:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.740123:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.740128:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.740132:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.740138:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.740143:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.740149:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.740154:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.740156:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.740160:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.740165:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.740170:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x100e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.740179:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.740184:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.740189:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.740196:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.740204:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.740212:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a380 -> ffff8fc4b8609918 00010000:00000001:0.0:1788573073.740219:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.740224:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.740231:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a380 -> ffff8fc4b8609918 00000020:00000001:0.0:1788573073.740238:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.740242:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.740248:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.740253:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.740259:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.740267:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a380 -> ffff8fc4b8609918 00000020:00000001:0.0:1788573073.740274:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8609918 -> ffff8fc4b8609918 00000020:00000001:0.0:1788573073.740279:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.740285:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.740293:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8f8a380 -> ffff8fc4b8609918 00000008:00000001:0.0:1788573073.740298:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.740313:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.740320:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.740328:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.740334:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.740340:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.740345:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.740352:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.740361:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a731fc00 with handle 0x7d52aadf1286c981 from hash 00010000:00000001:0.0:1788573073.740368:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.740373:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.740377:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.740381:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.740387:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.740393:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.740398:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573073.740403:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000edec7bc5/0x7d52aadf1286c965 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x53da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x784a792ceaa5a60c expref: -99 pid: 259981 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00000001:0.0:1788573073.740422:0:241699:0:(osc_lock.c:529:osc_ldlm_blocking_ast()) Process entered 00000020:00000001:0.0:1788573073.740425:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.740428:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806298216 : -123399903253400 : ffff8fc4b899ec68) 00000020:00001000:0.0:1788573073.740434:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.740437:0:241699:0:(osc_lock.c:413:__osc_dlm_blocking_ast()) Process entered 00000008:00000001:0.0:1788573073.740441:0:241699:0:(osc_lock.c:355:osc_lock_flush()) Process entered 00000020:00000001:0.0:1788573073.740444:0:241699:0:(cl_object.c:754:cl_env_obtain()) Process entered 00000020:00000001:0.0:1788573073.740448:0:241699:0:(cl_object.c:785:cl_env_obtain()) Process leaving (rc=18446620673806296200 : -123399903255416 : ffff8fc4b899e488) 00000020:00001000:0.0:1788573073.740463:0:241699:0:(cl_object.c:823:cl_env_get()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.740467:0:241699:0:(osc_cache.c:3760:osc_ldlm_hp_handle()) Process entered 00000008:00000020:0.0:1788573073.740470:0:241699:0:(osc_cache.c:3780:osc_ldlm_hp_handle()) High prio I/O check: start 0 end 4503599627370495 RPC(0):r0/w0/d0 00000008:00000001:0.0:1788573073.740473:0:241699:0:(osc_cache.c:3796:osc_ldlm_hp_handle()) Process leaving via out_unlock (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788573073.740478:0:241699:0:(osc_cache.c:3844:osc_ldlm_hp_handle()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.740482:0:241699:0:(osc_cache.c:3729:osc_lock_discard_pages()) Process entered 00000020:00000001:0.0:1788573073.740485:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0e8c0 -> ffff8fc4b38d7148 00000020:00000001:0.0:1788573073.740489:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b38d7148 -> ffff8fc4b38d7148 00000020:00000001:0.0:1788573073.740493:0:241699:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:0.0:1788573073.740500:0:241699:0:(vvp_io.c:1821:vvp_io_init()) Process entered 00000080:00200000:0.0:1788573073.740504:0:241699:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b17:0x100c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:0.0:1788573073.740508:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.740512:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:0.0:1788573073.740514:0:241699:0:(vvp_io.c:1898:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:0.0:1788573073.740517:0:241699:0:(lov_object.c:1590:lov_io_init()) [0x200002b17:0x100c:0x0]io ffff8fc4aceb8bd0 type 8 ignore/verify layout 1/0 00020000:00000001:0.0:1788573073.740523:0:241699:0:(lov_io.c:2181:lov_io_init_composite()) Process entered 00020000:00000001:0.0:1788573073.740526:0:241699:0:(lov_io.c:721:lov_io_slice_init()) Process entered 00020000:00000001:0.0:1788573073.740529:0:241699:0:(lov_io.c:825:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:0.0:1788573073.740532:0:241699:0:(lov_io.c:206:lov_io_subio_init()) Process entered 00020000:00000001:0.0:1788573073.740533:0:241699:0:(lov_io.c:214:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.740534:0:241699:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:0.0:1788573073.740535:0:241699:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:0.0:1788573073.740536:0:241699:0:(lov_io.c:2194:lov_io_init_composite()) Process leaving 00000020:00000001:0.0:1788573073.740537:0:241699:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788573073.740539:0:241699:0:(osc_cache.c:3516:osc_page_gang_lookup()) Process entered 00000008:00000001:0.0:1788573073.740543:0:241699:0:(osc_cache.c:3593:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788573073.740548:0:241699:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:0.0:1788573073.740551:0:241699:0:(lov_io.c:900:lov_io_fini()) Process entered 00020000:00000001:0.0:1788573073.740553:0:241699:0:(lov_io.c:919:lov_io_fini()) Process leaving 00000080:00000001:0.0:1788573073.740558:0:241699:0:(vvp_io.c:296:vvp_io_fini()) Process entered 00000080:00200000:0.0:1788573073.740561:0:241699:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b17:0x100c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00000001:0.0:1788573073.740564:0:241699:0:(vvp_io.c:402:vvp_io_fini()) Process leaving 00000020:00000001:0.0:1788573073.740568:0:241699:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000008:00000001:0.0:1788573073.740571:0:241699:0:(osc_cache.c:3748:osc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.740575:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899e480 00000008:00000001:0.0:1788573073.740580:0:241699:0:(osc_lock.c:387:osc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.740582:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0e8c0 -> ffff8fc4b38d7148 00010000:00000001:0.0:1788573073.740586:0:241699:0:(ldlm_extent.c:945:ldlm_extent_shift_kms()) Process entered 00010000:00000001:0.0:1788573073.740589:0:241699:0:(ldlm_extent.c:1018:ldlm_extent_shift_kms()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.740593:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0e8c0 -> ffff8fc4b38d7148 00000020:00000001:0.0:1788573073.740597:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000008:00000020:0.0:1788573073.740600:0:241699:0:(osc_object.c:161:osc_attr_update()) set kms from 0to 0 00020000:00000001:0.0:1788573073.740603:0:241699:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:0.0:1788573073.740607:0:241699:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.740609:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.740615:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0e8c0 -> ffff8fc4b38d7148 00000020:00000001:0.0:1788573073.740619:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b38d7148 -> ffff8fc4b38d7148 00000020:00000001:0.0:1788573073.740622:0:241699:0:(cl_object.c:248:cl_object_attr_update()) Process entered 00000020:00000001:0.0:1788573073.740630:0:241699:0:(cl_object.c:261:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.740633:0:241699:0:(cl_object.c:143:cl_object_top()) ffff8fc4b8e0e8c0 -> ffff8fc4b38d7148 00000008:00000001:0.0:1788573073.740639:0:241699:0:(osc_lock.c:470:__osc_dlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:0.0:1788573073.740644:0:241699:0:(cl_object.c:917:cl_env_put()) 1@ffff8fc4b899ec60 00000008:00000001:0.0:1788573073.740646:0:241699:0:(osc_lock.c:569:osc_ldlm_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.740650:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573073.740654:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573073.740658:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573073.740661:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573073.740664:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573073.740668:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc484dd9800 with handle 0x7d52aadf1286c965 from hash 00010000:00000001:0.0:1788573073.740670:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573073.740673:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573073.740676:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.740679:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573073.740683:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573073.740686:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573073.740691:0:241699:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=36 : 36 : 24) 00010000:00000001:0.0:1788573073.740697:0:241699:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788573073.740701:0:241699:0:(ldlm_request.c:1527:ldlm_cli_cancel_req()) Process entered 00000100:00000010:0.0:1788573073.740784:0:241699:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3100. 00000020:00000040:0.0:1788573073.740792:0:241699:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000100:00000001:0.0:1788573073.740802:0:241699:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573073.740833:0:241699:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573073.740842:0:241699:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573073.740855:0:241699:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d400. 00000100:00000001:0.0:1788573073.740865:0:241699:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573073.740872:0:241699:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788573073.740883:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000045df0a51/0x7d52aadf1286cd39 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5af9e expref: -99 pid: 260122 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.740908:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000004c6fe2f8/0x7d52aadf1286cd1d lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5af58 expref: -99 pid: 260118 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.740932:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000005ecb62b7/0x7d52aadf1286cd01 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5af12 expref: -99 pid: 260114 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.740959:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000000799443e/0x7d52aadf1286cce5 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5aecc expref: -99 pid: 260110 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.740981:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000a38b0691/0x7d52aadf1286ccc9 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ae86 expref: -99 pid: 260106 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741003:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000048388a30/0x7d52aadf1286ccad lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ae40 expref: -99 pid: 260102 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741025:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000016ecdf39/0x7d52aadf1286cc91 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5adfa expref: -99 pid: 260098 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741052:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000002fdc9c51/0x7d52aadf1286cc75 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5adb4 expref: -99 pid: 260094 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741071:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000004f30a754/0x7d52aadf1286cc59 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ad6e expref: -99 pid: 260090 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741097:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000095f3ad41/0x7d52aadf1286cc3d lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ad28 expref: -99 pid: 260086 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741121:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000b2ae5e66/0x7d52aadf1286cc21 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ace2 expref: -99 pid: 260082 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741145:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000ed1865b8/0x7d52aadf1286cc05 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ac9c expref: -99 pid: 260078 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741171:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000bce5c18c/0x7d52aadf1286cbe9 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ac56 expref: -99 pid: 260074 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741194:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000141d6b03/0x7d52aadf1286cbcd lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ac10 expref: -99 pid: 260070 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741219:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000002de066b1/0x7d52aadf1286cbb1 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5abca expref: -99 pid: 260065 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741245:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000005acd2e4a/0x7d52aadf1286cb95 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ab84 expref: -99 pid: 260061 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741265:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000a2fd5097/0x7d52aadf1286cb79 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ab3e expref: -99 pid: 260057 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741286:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000005fb7b720/0x7d52aadf1286cb5d lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5aaf8 expref: -99 pid: 260053 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741312:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000a7f306f4/0x7d52aadf1286cb41 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5aab2 expref: -99 pid: 260049 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741338:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000115854c3/0x7d52aadf1286cb25 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5aa6c expref: -99 pid: 260045 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741365:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000001625ffb1/0x7d52aadf1286cb09 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5aa26 expref: -99 pid: 260041 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741402:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000062314496/0x7d52aadf1286caed lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a9e0 expref: -99 pid: 260037 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741425:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000094e5b9b9/0x7d52aadf1286cad1 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a99a expref: -99 pid: 260033 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741468:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000075c7a7b1/0x7d52aadf1286cab5 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a954 expref: -99 pid: 260029 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741490:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000007b4ef045/0x7d52aadf1286ca99 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a90e expref: -99 pid: 260025 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741509:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000057a2b159/0x7d52aadf1286ca7d lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a8c8 expref: -99 pid: 260021 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741527:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000060285844/0x7d52aadf1286ca61 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a882 expref: -99 pid: 260017 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741549:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000004ab82e27/0x7d52aadf1286ca45 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a83c expref: -99 pid: 260013 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741569:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000010cbf922/0x7d52aadf1286ca29 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a7f6 expref: -99 pid: 260009 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741587:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000067e14c26/0x7d52aadf1286ca0d lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a7b0 expref: -99 pid: 260005 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741616:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000e53a778e/0x7d52aadf1286c9f1 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a76a expref: -99 pid: 260001 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741637:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000e0e82645/0x7d52aadf1286c9d5 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a724 expref: -99 pid: 259997 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741651:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000993cbdd5/0x7d52aadf1286c9b9 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a6de expref: -99 pid: 259993 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741672:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000009b9e6238/0x7d52aadf1286c99d lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a698 expref: -99 pid: 259989 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741693:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000a45b0440/0x7d52aadf1286c981 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a652 expref: -99 pid: 259985 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741713:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000edec7bc5/0x7d52aadf1286c965 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x53da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a60c expref: -99 pid: 259981 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573073.741740:0:241699:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 36 locks packed 00010000:00000001:0.0:1788573073.741746:0:241699:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=36 : 36 : 24) 00000100:00000001:0.0:1788573073.741758:0:241699:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573073.741766:0:241699:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573073.741786:0:241699:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000053a4b590] to pc [ptlrpcd_00_01+1] req@ffff8fc4b80c3100 x1875440516526592/t0(0) o103->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:17/18 lens 600/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1788573073.741859:0:241699:0:(ldlm_request.c:1595:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788573073.741868:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.741881:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000045df0a51/0x7d52aadf1286cd39 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5af9e expref: -99 pid: 260122 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.741906:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6fb7a80. 00010000:00000040:0.0:1788573073.741918:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc487a13d00 count: 0 00010000:00000010:0.0:1788573073.741929:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc487a13600. 00010000:00000001:0.0:1788573073.741942:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.741946:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.741952:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000004c6fe2f8/0x7d52aadf1286cd1d lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5af58 expref: -99 pid: 260118 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.741982:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6fb76c0. 00010000:00000040:0.0:1788573073.741990:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc487a13100 count: 0 00010000:00000010:0.0:1788573073.741997:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc487a13b00. 00010000:00000001:0.0:1788573073.742008:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742013:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742018:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000005ecb62b7/0x7d52aadf1286cd01 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5af12 expref: -99 pid: 260114 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742034:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b87c4380. 00010000:00000040:0.0:1788573073.742038:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4b3c61a00 count: 0 00010000:00000010:0.0:1788573073.742045:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4b3c61800. 00010000:00000001:0.0:1788573073.742054:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742058:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742061:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000000799443e/0x7d52aadf1286cce5 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5aecc expref: -99 pid: 260110 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742078:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6fb7080. 00010000:00000040:0.0:1788573073.742086:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4829ec700 count: 0 00010000:00000010:0.0:1788573073.742091:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4829ec100. 00010000:00000001:0.0:1788573073.742106:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742110:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742114:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000a38b0691/0x7d52aadf1286ccc9 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ae86 expref: -99 pid: 260106 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742129:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b154d3c0. 00010000:00000040:0.0:1788573073.742132:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc48857d500 count: 0 00010000:00000010:0.0:1788573073.742135:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc48857d100. 00010000:00000001:0.0:1788573073.742142:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742145:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742149:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000048388a30/0x7d52aadf1286ccad lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ae40 expref: -99 pid: 260102 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742164:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b87c4c00. 00010000:00000040:0.0:1788573073.742171:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4829c3d00 count: 0 00010000:00000010:0.0:1788573073.742179:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4829c3300. 00010000:00000001:0.0:1788573073.742192:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742196:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742204:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000016ecdf39/0x7d52aadf1286cc91 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5adfa expref: -99 pid: 260098 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742223:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6fb7180. 00010000:00000040:0.0:1788573073.742230:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4829ecc00 count: 0 00010000:00000010:0.0:1788573073.742237:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4829ec900. 00010000:00000001:0.0:1788573073.742245:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742250:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742256:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000002fdc9c51/0x7d52aadf1286cc75 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5adb4 expref: -99 pid: 260094 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742279:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6fb7300. 00010000:00000040:0.0:1788573073.742286:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4829ec200 count: 0 00010000:00000010:0.0:1788573073.742301:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4829eca00. 00010000:00000001:0.0:1788573073.742313:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742319:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742325:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000004f30a754/0x7d52aadf1286cc59 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ad6e expref: -99 pid: 260090 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742344:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6f15e00. 00010000:00000040:0.0:1788573073.742351:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4b3947900 count: 0 00010000:00000010:0.0:1788573073.742359:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4b3947f00. 00010000:00000001:0.0:1788573073.742371:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742376:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742381:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000095f3ad41/0x7d52aadf1286cc3d lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ad28 expref: -99 pid: 260086 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742399:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b87c45c0. 00010000:00000040:0.0:1788573073.742403:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc483164b00 count: 0 00010000:00000010:0.0:1788573073.742407:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc483164400. 00010000:00000001:0.0:1788573073.742417:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742420:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742425:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000b2ae5e66/0x7d52aadf1286cc21 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ace2 expref: -99 pid: 260082 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742440:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6f154c0. 00010000:00000040:0.0:1788573073.742446:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc48679ba00 count: 0 00010000:00000010:0.0:1788573073.742450:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc48679b200. 00010000:00000001:0.0:1788573073.742469:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742471:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742474:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000ed1865b8/0x7d52aadf1286cc05 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ac9c expref: -99 pid: 260078 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742484:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b154db00. 00010000:00000040:0.0:1788573073.742496:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc48857d800 count: 0 00010000:00000010:0.0:1788573073.742500:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc48857d700. 00010000:00000001:0.0:1788573073.742506:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742509:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742513:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000bce5c18c/0x7d52aadf1286cbe9 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ac56 expref: -99 pid: 260074 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742528:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b87c4bc0. 00010000:00000040:0.0:1788573073.742532:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc483164800 count: 0 00010000:00000010:0.0:1788573073.742537:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc483164a00. 00010000:00000001:0.0:1788573073.742544:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742546:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742550:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000141d6b03/0x7d52aadf1286cbcd lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ac10 expref: -99 pid: 260070 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742560:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b87c4c80. 00010000:00000040:0.0:1788573073.742563:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc483164e00 count: 0 00010000:00000010:0.0:1788573073.742567:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc483164600. 00010000:00000001:0.0:1788573073.742574:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742576:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742581:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000002de066b1/0x7d52aadf1286cbb1 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5abca expref: -99 pid: 260065 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742592:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6fb74c0. 00010000:00000040:0.0:1788573073.742597:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4829ecf00 count: 0 00010000:00000010:0.0:1788573073.742603:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4829ec800. 00010000:00000001:0.0:1788573073.742608:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742611:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742613:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000005acd2e4a/0x7d52aadf1286cb95 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ab84 expref: -99 pid: 260061 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742629:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6fb7b80. 00010000:00000040:0.0:1788573073.742635:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4829ece00 count: 0 00010000:00000010:0.0:1788573073.742642:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4829ec300. 00010000:00000001:0.0:1788573073.742651:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742655:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742662:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000a2fd5097/0x7d52aadf1286cb79 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5ab3e expref: -99 pid: 260057 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742685:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6fb7880. 00010000:00000040:0.0:1788573073.742693:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc488a73100 count: 0 00010000:00000010:0.0:1788573073.742700:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4829ec500. 00010000:00000001:0.0:1788573073.742711:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742716:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742723:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000005fb7b720/0x7d52aadf1286cb5d lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5aaf8 expref: -99 pid: 260053 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742747:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6fb7d80. 00010000:00000040:0.0:1788573073.742751:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc488a73300 count: 0 00010000:00000010:0.0:1788573073.742757:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc488a73c00. 00010000:00000001:0.0:1788573073.742766:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742770:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742775:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000a7f306f4/0x7d52aadf1286cb41 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5aab2 expref: -99 pid: 260049 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742790:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6fb7340. 00010000:00000040:0.0:1788573073.742796:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4904c6000 count: 0 00010000:00000010:0.0:1788573073.742801:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc488a73500. 00010000:00000001:0.0:1788573073.742840:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742851:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742855:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000115854c3/0x7d52aadf1286cb25 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5aa6c expref: -99 pid: 260045 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742872:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b82bd880. 00010000:00000040:0.0:1788573073.742881:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4904c6b00 count: 0 00010000:00000010:0.0:1788573073.742890:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4904c6800. 00010000:00000001:0.0:1788573073.742903:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742908:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742915:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000001625ffb1/0x7d52aadf1286cb09 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5aa26 expref: -99 pid: 260041 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742932:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b82bd800. 00010000:00000040:0.0:1788573073.742938:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4904c6500 count: 0 00010000:00000010:0.0:1788573073.742944:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4904c6900. 00010000:00000001:0.0:1788573073.742951:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742955:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742960:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000062314496/0x7d52aadf1286caed lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a9e0 expref: -99 pid: 260037 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.742973:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6f150c0. 00010000:00000040:0.0:1788573073.742976:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4b820a800 count: 0 00010000:00000010:0.0:1788573073.742980:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4b820a700. 00010000:00000001:0.0:1788573073.742986:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.742988:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.742993:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000094e5b9b9/0x7d52aadf1286cad1 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a99a expref: -99 pid: 260033 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743009:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b82bdc00. 00010000:00000040:0.0:1788573073.743015:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4904c6c00 count: 0 00010000:00000010:0.0:1788573073.743019:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4904c6d00. 00010000:00000001:0.0:1788573073.743034:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743037:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743041:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000075c7a7b1/0x7d52aadf1286cab5 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a954 expref: -99 pid: 260029 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743058:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4a6f15500. 00010000:00000040:0.0:1788573073.743063:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4b820a000 count: 0 00010000:00000010:0.0:1788573073.743068:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4b820a200. 00010000:00000001:0.0:1788573073.743076:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743079:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743083:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000007b4ef045/0x7d52aadf1286ca99 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a90e expref: -99 pid: 260025 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743090:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b82bd6c0. 00010000:00000040:0.0:1788573073.743093:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4904c6700 count: 0 00010000:00000010:0.0:1788573073.743097:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4904c6400. 00010000:00000001:0.0:1788573073.743103:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743107:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743112:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000057a2b159/0x7d52aadf1286ca7d lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a8c8 expref: -99 pid: 260021 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743126:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b82bd440. 00010000:00000040:0.0:1788573073.743132:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc487b43800 count: 0 00010000:00000010:0.0:1788573073.743137:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4904c6100. 00010000:00000001:0.0:1788573073.743144:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743147:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743151:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000060285844/0x7d52aadf1286ca61 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a882 expref: -99 pid: 260017 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743167:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b87c4340. 00010000:00000040:0.0:1788573073.743173:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4a6bc5900 count: 0 00010000:00000010:0.0:1788573073.743185:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4a6bc5000. 00010000:00000001:0.0:1788573073.743195:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743197:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743199:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000004ab82e27/0x7d52aadf1286ca45 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a83c expref: -99 pid: 260013 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743209:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b82bd4c0. 00010000:00000040:0.0:1788573073.743214:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc487b43100 count: 0 00010000:00000010:0.0:1788573073.743218:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc487b43b00. 00010000:00000001:0.0:1788573073.743226:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743228:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743233:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000010cbf922/0x7d52aadf1286ca29 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a7f6 expref: -99 pid: 260009 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743248:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b17402c0. 00010000:00000040:0.0:1788573073.743254:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4b820af00 count: 0 00010000:00000010:0.0:1788573073.743258:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4b820a100. 00010000:00000001:0.0:1788573073.743266:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743270:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743275:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 0000000067e14c26/0x7d52aadf1286ca0d lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a7b0 expref: -99 pid: 260005 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743290:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b82bd040. 00010000:00000040:0.0:1788573073.743293:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc487b43900 count: 0 00010000:00000010:0.0:1788573073.743298:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc487b43700. 00010000:00000001:0.0:1788573073.743305:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743309:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743314:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000e53a778e/0x7d52aadf1286c9f1 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a76a expref: -99 pid: 260001 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743327:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b1740740. 00010000:00000040:0.0:1788573073.743338:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc48973ca00 count: 0 00010000:00000010:0.0:1788573073.743345:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc48973c000. 00010000:00000001:0.0:1788573073.743353:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743357:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743362:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000e0e82645/0x7d52aadf1286c9d5 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a724 expref: -99 pid: 259997 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743376:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b87c4540. 00010000:00000040:0.0:1788573073.743380:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4a6bc5d00 count: 0 00010000:00000010:0.0:1788573073.743385:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc4a6bc5c00. 00010000:00000001:0.0:1788573073.743393:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743396:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743401:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000993cbdd5/0x7d52aadf1286c9b9 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a6de expref: -99 pid: 259993 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743419:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b82bda80. 00010000:00000040:0.0:1788573073.743425:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc488404300 count: 0 00010000:00000010:0.0:1788573073.743432:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc487b43600. 00010000:00000001:0.0:1788573073.743443:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743447:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743463:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 000000009b9e6238/0x7d52aadf1286c99d lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a698 expref: -99 pid: 259989 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743484:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b82bd380. 00010000:00000040:0.0:1788573073.743492:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc488404b00 count: 0 00010000:00000010:0.0:1788573073.743500:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc488404400. 00010000:00000001:0.0:1788573073.743509:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743512:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743516:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000a45b0440/0x7d52aadf1286c981 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a652 expref: -99 pid: 259985 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743549:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b82bd140. 00010000:00000040:0.0:1788573073.743552:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc488404f00 count: 0 00010000:00000010:0.0:1788573073.743557:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc488404500. 00010000:00000001:0.0:1788573073.743563:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743566:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573073.743569:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8fc490e72800 lock: 00000000edec7bc5/0x7d52aadf1286c965 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x53da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x784a792ceaa5a60c expref: -99 pid: 259981 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00000010:0.0:1788573073.743573:0:241699:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 56 at ffff8fc4b1740800. 00010000:00000040:0.0:1788573073.743578:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc48973c600 count: 0 00010000:00000010:0.0:1788573073.743583:0:241699:0:(ldlm_resource.c:1563:ldlm_resource_free()) slab-freed 'res->lr_itree': 216 at ffff8fc48973c800. 00010000:00000001:0.0:1788573073.743589:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573073.743591:0:241699:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.0:1788573073.743595:0:241699:0:(ldlm_lockd.c:2919:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff8fc485941d80. 00010000:00000001:0.0:1788573073.743600:0:241699:0:(ldlm_lockd.c:2923:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573073.743605:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573073.743608:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573073.743612:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573073.743614:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573073.743616:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573073.743619:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573073.743624:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573073.743628:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573073.743634:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573074.048909:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573074.048919:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573074.048925:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573074.048934:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573074.048943:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573074.048951:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573074.048967:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573074.048980:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573074.048983:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573075.072943:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573075.072950:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573075.072954:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573075.072959:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573075.072964:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573075.072970:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.072974:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573075.072978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573075.072981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573075.328904:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308435181, at 4308435200 00000100:00000010:0.0:1788573075.328932:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea79c0. 00000100:00000001:0.0:1788573076.096914:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573076.096927:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573076.096932:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573076.096938:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573076.096945:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573076.096950:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573076.096959:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573076.096965:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573076.096973:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573077.120909:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573077.120921:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573077.120925:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573077.120936:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573077.120943:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573077.120947:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573077.120953:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573077.120957:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573077.120963:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573077.120992:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573077.120997:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573077.120999:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573077.121001:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573077.121003:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573077.121006:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573077.121013:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573077.121015:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573077.121017:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573077.825013:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573077.825016:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573077.825019:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573078.144894:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573078.144909:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573078.144913:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573078.144918:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573078.144926:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573078.144931:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573078.144954:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573078.144963:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573078.144966:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573079.168915:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573079.168924:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573079.168929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573079.168939:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573079.168948:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573079.168955:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573079.168959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573079.168964:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573079.168966:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573080.192885:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573080.192892:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573080.192894:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573080.192900:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573080.192905:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573080.192911:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573080.192924:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573080.192928:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573080.192934:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573080.448896:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308440301, at 4308440320 00000100:00000010:0.0:1788573080.448912:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7540. 00000100:00000001:0.0:1788573081.216949:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573081.216955:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573081.216975:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573081.216982:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573081.216986:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573081.216991:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573081.216997:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573081.216999:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573081.217002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.217101:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573081.217104:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573081.217105:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573081.217107:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573081.217110:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573081.217112:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573081.217114:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573081.217118:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573081.217120:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573081.728916:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573081.728930:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573081.728934:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573081.728939:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573081.728945:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573081.728954:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13769 00000400:00000200:0.0:1788573081.728964:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573081.728970:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573081.728973:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573081.728975:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.728978:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573081.728980:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.728982:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573081.728985:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573081.728986:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573081.728988:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573081.728991:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573081.728992:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573081.729010:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573081.729012:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729014:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573081.729015:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729017:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573081.729019:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573081.729021:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729023:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573081.729025:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573081.729027:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573081.729029:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729031:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573081.729033:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573081.729037:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573081.729040:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573081.729043:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573081.729045:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729047:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573081.729049:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573081.729053:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573081.729058:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573081.729063:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573081.729070:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516527488/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573081.729078:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573081.729082:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573081.729085:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516527488/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573081.729103:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573081.729105:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573081.729108:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573081.729112:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573081.729114:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573081.729117:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573081.729120:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13769 00000400:00000200:0.0:1788573081.729124:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573081.729129:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573081.729132:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573081.729137:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729139:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573081.729143:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729160:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573081.729165:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573081.729167:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573081.729170:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573081.729172:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573081.729176:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573081.729180:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573081.729184:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729187:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573081.729191:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729196:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573081.729201:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573081.729204:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729205:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573081.729206:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573081.729213:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573081.729217:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729221:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573081.729224:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573081.729229:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729238:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573081.729246:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573081.729251:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573081.729256:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573081.729260:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729265:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573081.729269:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573081.729274:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573081.729281:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573081.729287:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573081.729297:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516527616/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573081.729311:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573081.729313:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573081.729320:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516527616/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573081.729330:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573081.729334:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573081.729337:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573081.729340:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573081.729344:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573081.729348:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573081.729354:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13769 00000400:00000200:0.0:1788573081.729360:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573081.729366:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573081.729371:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573081.729377:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729382:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573081.729386:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729389:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573081.729391:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573081.729400:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573081.729404:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573081.729408:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573081.729411:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573081.729416:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573081.729421:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729423:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573081.729425:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729426:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573081.729430:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573081.729433:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729438:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573081.729440:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573081.729444:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573081.729449:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729461:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573081.729463:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573081.729467:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729470:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573081.729476:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573081.729479:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573081.729482:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573081.729483:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729484:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573081.729487:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573081.729490:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573081.729495:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573081.729498:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573081.729505:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516527744/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573081.729525:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573081.729528:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573081.729542:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516527744/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573081.729551:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573081.729555:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573081.729560:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573081.729564:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573081.729566:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573081.729568:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573081.729573:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13769 00000400:00000200:0.0:1788573081.729577:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573081.729583:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573081.729585:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573081.729591:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729595:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573081.729599:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729608:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573081.729611:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573081.729614:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573081.729617:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573081.729620:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573081.729624:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573081.729628:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573081.729633:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729635:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573081.729637:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729638:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573081.729642:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573081.729648:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729653:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573081.729661:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573081.729665:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573081.729670:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729673:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573081.729676:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573081.729681:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729685:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573081.729689:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573081.729693:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573081.729698:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573081.729701:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729705:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573081.729708:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573081.729711:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573081.729716:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573081.729723:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573081.729730:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516527872/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573081.729752:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573081.729758:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573081.729768:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516527872/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573081.729803:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573081.729831:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573081.729838:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573081.729841:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573081.729844:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573081.729850:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573081.729855:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13769 00000400:00000200:0.0:1788573081.729861:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573081.729865:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573081.729870:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573081.729881:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729884:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573081.729889:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729891:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573081.729893:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573081.729897:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573081.729901:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573081.729903:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573081.729908:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573081.729911:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573081.729916:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729921:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573081.729926:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729929:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573081.729933:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573081.729940:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573081.729947:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573081.729951:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573081.729957:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573081.729962:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729967:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573081.729969:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573081.729974:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729976:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573081.729980:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573081.729984:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573081.729988:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573081.729991:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573081.729993:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573081.729995:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573081.729999:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573081.730004:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573081.730013:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573081.730022:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516528000/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573081.730035:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573081.730039:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573081.730044:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516528000/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573081.730060:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573081.730065:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13779) 00000100:00000001:0.0:1788573082.240940:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573082.240946:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573082.240950:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573082.240956:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573082.240962:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573082.240967:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573082.240982:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573082.240988:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573082.240992:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573082.944961:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573082.944964:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573082.944967:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573083.265356:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573083.265360:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573083.265362:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573083.265366:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573083.265371:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573083.265377:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573083.265382:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573083.265388:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573083.265392:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573084.288964:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573084.288971:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573084.288976:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573084.288982:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573084.288988:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573084.289001:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573084.289010:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573084.289015:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573084.289021:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573085.312934:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573085.312940:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573085.312946:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573085.312954:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573085.312960:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573085.312966:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573085.312971:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573085.312975:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573085.312980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573085.568892:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308445421, at 4308445440 00000100:00000010:0.0:1788573085.568908:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea75a0. 00000100:00000001:0.0:1788573086.336933:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573086.336944:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573086.336948:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573086.336964:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573086.336984:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573086.336988:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573086.337001:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573086.337004:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573086.337006:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.337097:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573086.337101:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573086.337103:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573086.337107:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573086.337110:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573086.337124:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573086.337129:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573086.337131:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573086.337136:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573086.848880:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573086.848892:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573086.848895:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573086.848918:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573086.848922:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573086.848928:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13774 00000400:00000200:0.0:1788573086.848936:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573086.848941:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573086.848943:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573086.848946:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.848972:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.848974:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.848976:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573086.848980:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573086.848981:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573086.848983:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573086.848986:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573086.848987:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573086.848989:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573086.848991:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.848993:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.848994:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.848996:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573086.848998:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573086.849000:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849001:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573086.849003:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573086.849006:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573086.849008:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849009:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573086.849011:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573086.849017:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573086.849019:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573086.849022:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573086.849029:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849032:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573086.849034:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573086.849038:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573086.849044:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573086.849050:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573086.849058:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516528128/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573086.849066:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573086.849069:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573086.849073:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516528128/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573086.849090:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573086.849092:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573086.849095:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573086.849096:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573086.849097:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573086.849099:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573086.849101:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13774 00000400:00000200:0.0:1788573086.849104:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573086.849107:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573086.849110:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573086.849112:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849114:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.849119:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849121:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573086.849125:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573086.849127:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573086.849129:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573086.849132:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573086.849134:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573086.849143:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573086.849150:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849154:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.849157:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849162:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573086.849168:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573086.849177:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849183:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573086.849185:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573086.849193:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573086.849199:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849201:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573086.849205:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573086.849207:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849208:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573086.849211:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573086.849215:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573086.849218:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573086.849219:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849224:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573086.849227:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573086.849229:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573086.849236:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573086.849240:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573086.849246:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516528256/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573086.849259:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573086.849263:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573086.849271:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516528256/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573086.849288:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573086.849299:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573086.849303:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573086.849305:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573086.849308:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573086.849311:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573086.849314:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13774 00000400:00000200:0.0:1788573086.849321:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573086.849327:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573086.849329:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573086.849334:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849337:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.849341:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849344:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573086.849347:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573086.849349:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573086.849352:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573086.849354:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573086.849357:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573086.849360:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573086.849368:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849375:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.849380:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849387:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573086.849395:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573086.849403:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849407:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573086.849412:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573086.849416:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573086.849420:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849424:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573086.849433:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573086.849438:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849442:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573086.849449:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573086.849454:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573086.849459:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573086.849462:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849466:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573086.849470:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573086.849475:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573086.849482:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573086.849487:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573086.849493:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516528384/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573086.849501:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573086.849504:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573086.849510:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516528384/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573086.849542:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573086.849546:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573086.849550:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573086.849553:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573086.849555:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573086.849559:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573086.849564:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13774 00000400:00000200:0.0:1788573086.849572:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573086.849577:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573086.849583:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573086.849588:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849589:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.849591:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849599:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573086.849604:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573086.849607:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573086.849611:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573086.849614:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573086.849619:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573086.849620:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573086.849624:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849628:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.849630:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849632:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573086.849638:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573086.849645:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849651:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573086.849653:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573086.849658:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573086.849661:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849664:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573086.849668:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573086.849674:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849678:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573086.849684:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573086.849687:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573086.849691:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573086.849695:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849700:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573086.849704:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573086.849707:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573086.849711:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573086.849715:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573086.849724:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516528512/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573086.849749:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573086.849753:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573086.849761:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516528512/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573086.849790:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573086.849796:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573086.849801:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573086.849802:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573086.849805:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573086.849826:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573086.849828:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13774 00000400:00000200:0.0:1788573086.849832:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573086.849834:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573086.849836:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573086.849843:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849848:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.849850:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849856:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573086.849862:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573086.849866:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573086.849870:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573086.849874:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573086.849879:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573086.849884:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573086.849891:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849894:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573086.849898:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849901:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573086.849912:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573086.849919:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573086.849926:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573086.849929:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573086.849932:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573086.849938:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849943:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573086.849946:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573086.849960:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849961:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573086.849964:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573086.849966:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573086.849969:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573086.849970:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573086.849971:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573086.849972:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573086.849974:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573086.849981:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573086.849986:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573086.849995:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516528640/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573086.850006:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573086.850009:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573086.850016:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516528640/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573086.850027:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573086.850032:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13784) 00000100:00000001:0.0:1788573087.360947:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573087.360957:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573087.360963:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573087.360981:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573087.360988:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573087.361006:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573087.361015:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573087.361019:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573087.361022:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573088.065019:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573088.065022:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573088.065025:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573088.384939:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573088.384949:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573088.384953:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573088.384961:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573088.384969:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573088.384974:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573088.384996:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573088.384999:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573088.385002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573089.408956:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573089.408961:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573089.408964:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573089.408970:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573089.408976:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573089.408978:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573089.408980:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573089.408982:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573089.408984:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573090.432958:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573090.432962:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573090.432966:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573090.432970:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573090.432976:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573090.432980:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573090.432983:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573090.432984:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573090.432986:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573090.688928:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308450541, at 4308450560 00000100:00000010:0.0:1788573090.688950:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7360. 00000100:00000001:0.0:1788573091.456929:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573091.456953:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573091.456958:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573091.456966:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573091.456971:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573091.456976:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573091.456995:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573091.456999:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573091.457003:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573091.968892:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573091.968903:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573091.968907:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573091.968910:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573091.968916:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573091.968924:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13779 00000400:00000200:0.0:1788573091.968934:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573091.968941:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573091.968945:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573091.968948:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.968951:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573091.968953:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.968955:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573091.968958:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573091.968959:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573091.968962:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573091.968964:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573091.968966:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573091.968967:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573091.968970:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.968971:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573091.968973:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.968974:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573091.968976:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573091.968995:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573091.968997:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573091.968999:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573091.969002:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573091.969004:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969006:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573091.969008:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573091.969013:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788573091.969016:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573091.969019:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573091.969020:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969022:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573091.969025:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573091.969028:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573091.969034:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573091.969040:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573091.969047:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516528768/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573091.969055:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573091.969059:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573091.969062:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516528768/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573091.969081:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573091.969084:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573091.969086:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573091.969087:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573091.969089:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573091.969090:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573091.969093:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13779 00000400:00000200:0.0:1788573091.969096:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573091.969099:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573091.969101:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573091.969110:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969114:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573091.969117:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969121:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573091.969123:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573091.969126:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573091.969129:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573091.969132:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573091.969136:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573091.969139:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573091.969145:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969149:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573091.969153:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969157:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573091.969162:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573091.969170:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969173:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573091.969176:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573091.969183:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573091.969187:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969191:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573091.969195:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573091.969200:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969203:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573091.969208:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573091.969210:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573091.969213:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573091.969215:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969217:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573091.969218:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573091.969220:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573091.969226:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573091.969236:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573091.969243:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516528896/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573091.969272:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573091.969277:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573091.969282:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516528896/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573091.969295:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573091.969298:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573091.969302:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573091.969303:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573091.969304:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573091.969307:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573091.969311:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13779 00000400:00000200:0.0:1788573091.969319:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573091.969326:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573091.969333:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573091.969343:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969349:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573091.969356:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969360:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573091.969366:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573091.969370:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573091.969375:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573091.969380:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573091.969386:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573091.969391:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573091.969400:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969406:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573091.969420:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969426:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573091.969431:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573091.969436:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969442:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573091.969445:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573091.969450:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573091.969451:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969453:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573091.969455:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573091.969459:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969463:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573091.969468:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573091.969472:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573091.969476:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573091.969479:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969484:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573091.969487:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573091.969490:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573091.969494:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573091.969498:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573091.969507:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516529024/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573091.969516:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573091.969520:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573091.969525:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516529024/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573091.969536:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573091.969539:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573091.969542:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573091.969546:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573091.969554:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573091.969558:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573091.969572:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13779 00000400:00000200:0.0:1788573091.969583:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573091.969591:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573091.969596:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573091.969605:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969612:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573091.969620:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969627:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573091.969634:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573091.969638:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573091.969643:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573091.969646:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573091.969652:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573091.969657:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573091.969664:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969668:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573091.969673:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969678:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573091.969682:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573091.969687:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969691:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573091.969694:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573091.969698:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573091.969704:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969708:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573091.969712:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573091.969716:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969720:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573091.969733:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573091.969737:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573091.969741:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573091.969744:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969748:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573091.969750:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573091.969768:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573091.969774:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573091.969780:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573091.969787:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516529152/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573091.969796:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573091.969799:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573091.969804:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516529152/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573091.969831:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573091.969834:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573091.969838:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573091.969841:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573091.969842:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573091.969845:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573091.969848:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13779 00000400:00000200:0.0:1788573091.969854:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573091.969859:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573091.969863:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573091.969869:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969873:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573091.969874:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969876:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573091.969880:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573091.969888:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573091.969891:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573091.969895:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573091.969900:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573091.969906:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573091.969916:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969923:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573091.969931:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969938:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573091.969943:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573091.969951:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573091.969956:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573091.969959:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573091.969962:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573091.969968:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969972:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573091.969976:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573091.969979:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.969981:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573091.969987:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573091.969991:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573091.969997:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573091.969999:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573091.970000:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573091.970003:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573091.970006:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573091.970012:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573091.970019:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573091.970026:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516529280/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573091.970043:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573091.970048:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573091.970062:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516529280/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573091.970084:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573091.970092:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13789) 00000100:00000001:0.0:1788573092.481012:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573092.481018:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573092.481021:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573092.481025:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573092.481028:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573092.481030:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573092.481031:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573092.481033:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573092.481035:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573093.185008:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573093.185011:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573093.185014:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573093.504882:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573093.504889:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573093.504893:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573093.504901:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573093.504908:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573093.504914:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573093.504918:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573093.504921:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573093.504925:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573093.505006:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573093.505009:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573093.505011:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573093.505013:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573093.505016:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573093.505017:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573093.505019:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573093.505020:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573093.505022:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573094.528937:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573094.528943:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573094.528949:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573094.528971:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573094.528976:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573094.528982:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573094.528994:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573094.528997:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573094.529001:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573095.552901:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573095.552905:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573095.552909:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573095.552916:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573095.552921:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573095.552924:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573095.552930:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573095.552933:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573095.552937:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573095.808881:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308455661, at 4308455680 00000100:00000010:0.0:1788573095.808896:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7b40. 00000100:00000001:0.0:1788573096.576913:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573096.576923:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573096.576925:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573096.576931:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573096.576939:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573096.576945:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573096.576949:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573096.576955:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573096.576988:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573097.088907:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573097.088920:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573097.088923:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573097.088928:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573097.088934:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573097.088943:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13784 00000400:00000200:0.0:1788573097.088952:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573097.088959:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573097.088962:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573097.088982:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.088984:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.088986:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.088988:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573097.088992:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573097.088993:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573097.088995:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573097.088998:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573097.088999:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573097.089001:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573097.089003:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089005:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.089006:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089007:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573097.089010:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573097.089012:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089014:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573097.089016:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573097.089019:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573097.089021:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089022:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573097.089024:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573097.089029:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573097.089031:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573097.089034:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573097.089035:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089037:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573097.089040:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573097.089044:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573097.089051:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573097.089058:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573097.089067:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516529408/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573097.089085:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573097.089089:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573097.089095:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516529408/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573097.089119:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573097.089121:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573097.089124:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573097.089125:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573097.089127:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573097.089129:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573097.089132:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13784 00000400:00000200:0.0:1788573097.089136:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573097.089139:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573097.089141:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573097.089146:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089150:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.089154:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089156:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573097.089159:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573097.089161:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573097.089164:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573097.089167:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573097.089171:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573097.089176:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573097.089181:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089185:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.089189:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089193:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573097.089202:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573097.089210:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089215:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573097.089217:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573097.089224:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573097.089228:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089234:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573097.089238:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573097.089245:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089248:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573097.089253:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573097.089256:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573097.089259:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573097.089262:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089263:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573097.089264:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573097.089266:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573097.089271:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573097.089276:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573097.089285:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516529536/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573097.089300:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573097.089304:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573097.089309:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516529536/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573097.089318:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573097.089321:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573097.089326:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573097.089327:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573097.089329:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573097.089334:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573097.089346:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13784 00000400:00000200:0.0:1788573097.089353:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573097.089359:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573097.089361:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573097.089367:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089369:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.089372:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089377:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573097.089379:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573097.089381:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573097.089383:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573097.089386:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573097.089388:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573097.089391:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573097.089399:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089403:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.089408:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089413:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573097.089416:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573097.089421:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089424:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573097.089426:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573097.089428:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573097.089433:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089437:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573097.089440:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573097.089443:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089446:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573097.089461:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573097.089466:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573097.089474:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573097.089482:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089485:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573097.089487:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573097.089492:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573097.089499:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573097.089505:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573097.089516:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516529664/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573097.089533:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573097.089538:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573097.089570:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516529664/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573097.089582:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573097.089587:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573097.089593:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573097.089595:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573097.089598:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573097.089601:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573097.089604:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13784 00000400:00000200:0.0:1788573097.089616:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573097.089622:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573097.089625:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573097.089627:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089628:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.089631:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089635:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573097.089636:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573097.089638:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573097.089641:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573097.089650:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573097.089654:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573097.089664:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573097.089668:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089673:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.089676:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089681:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573097.089683:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573097.089689:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089694:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573097.089697:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573097.089700:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573097.089703:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089708:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573097.089711:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573097.089716:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089718:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573097.089722:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573097.089727:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573097.089733:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573097.089735:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089738:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573097.089742:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573097.089744:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573097.089750:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573097.089756:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573097.089765:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516529792/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573097.089776:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573097.089780:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573097.089787:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516529792/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573097.089837:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573097.089844:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573097.089848:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573097.089850:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573097.089852:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573097.089856:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573097.089861:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13784 00000400:00000200:0.0:1788573097.089869:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573097.089875:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573097.089878:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573097.089885:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089888:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.089894:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089901:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573097.089906:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573097.089909:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573097.089913:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573097.089916:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573097.089921:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573097.089926:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573097.089936:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089942:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573097.089947:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089950:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573097.089954:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573097.089957:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573097.089961:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573097.089962:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573097.089964:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573097.089965:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089967:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573097.089976:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573097.089980:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.089983:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573097.089988:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573097.089991:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573097.089993:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573097.089996:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573097.090000:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573097.090003:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573097.090007:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573097.090013:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573097.090017:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573097.090023:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516529920/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573097.090030:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573097.090034:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573097.090039:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516529920/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573097.090052:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573097.090059:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13794) 00000100:00000001:0.0:1788573097.600930:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573097.600936:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573097.600938:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573097.600941:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573097.600946:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573097.600956:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573097.600959:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573097.600961:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573097.600964:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573098.305948:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573098.305953:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573098.305957:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573098.624921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573098.624926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573098.624944:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573098.624947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573098.624949:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573098.624952:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573098.624974:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573098.624978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573098.624982:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573099.648898:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573099.648902:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573099.648906:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573099.648911:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573099.648919:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573099.648924:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573099.648946:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573099.648948:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573099.648950:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573100.672882:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573100.672891:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573100.672896:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573100.672902:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573100.672908:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573100.672914:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573100.672921:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573100.672925:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573100.672932:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573100.928912:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308460781, at 4308460800 00000100:00000010:0.0:1788573100.928928:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7600. 00000100:00000001:0.0:1788573101.696922:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573101.696933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573101.696939:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573101.696949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573101.696958:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573101.696963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573101.696973:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573101.696976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573101.696979:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.720922:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573102.720943:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573102.720946:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573102.720949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573102.720951:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573102.720954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573102.720955:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573102.720957:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573102.720959:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573102.721087:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573102.721092:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573102.721096:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573102.721103:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573102.721109:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573102.721114:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573102.721116:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573102.721117:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573102.721119:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573103.425028:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573103.425031:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573103.425034:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573103.744949:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573103.744953:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573103.744958:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573103.744963:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573103.744968:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573103.744972:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573103.744990:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573103.744994:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573103.744997:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573104.768919:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573104.768925:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573104.768930:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573104.768938:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573104.768943:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573104.768946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573104.768948:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573104.768950:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573104.768953:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788573105.281117:0:2387:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788573105.281170:0:2387:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788573105.281192:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573105.281196:0:2387:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788573105.281202:0:2387:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573105.281205:0:2387:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788573105.281211:0:2387:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573105.281215:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793900. 00000400:00000200:0.0:1788573105.281221:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573105.281227:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573105.281235:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc6e8 00000400:00000010:0.0:1788573105.281239:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc6e8. 00000100:00000001:0.0:1788573105.281260:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573105.281269:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac820e00 x1875440516530048/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573105.281298:0:2387:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573105.281304:0:2387:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516530048/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573127 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573105.281313:0:2387:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573105.281327:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573105.281337:0:2387:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573105.281340:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793b00. 00000400:00000200:0.0:1788573105.281343:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573105.281347:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573105.281351:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc330 00000400:00000010:0.0:1788573105.281353:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc330. 00000100:00000001:0.0:1788573105.281355:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573105.281360:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac821c00 x1875440516530176/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573105.281371:0:2387:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573105.281375:0:2387:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516530176/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573127 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573105.281422:0:2387:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573105.281427:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573105.281437:0:2387:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573105.281443:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793400. 00000400:00000200:0.0:1788573105.281450:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573105.281459:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573105.281465:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc088 00000400:00000010:0.0:1788573105.281468:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc088. 00000100:00000001:0.0:1788573105.281473:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573105.281483:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac823800 x1875440516530304/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573105.281495:0:2387:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573105.281502:0:2387:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516530304/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573127 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573105.281525:0:2387:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573105.281530:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573105.281539:0:2387:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573105.281544:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793500. 00000400:00000200:0.0:1788573105.281549:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573105.281557:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573105.281563:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc880 00000400:00000010:0.0:1788573105.281567:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc880. 00000100:00000001:0.0:1788573105.281572:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573105.281579:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac821880 x1875440516530432/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573105.281605:0:2387:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573105.281613:0:2387:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516530432/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573127 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573105.281631:0:2387:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573105.281635:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573105.281653:0:2387:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573105.281658:0:2387:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793d00. 00000400:00000200:0.0:1788573105.281664:0:2387:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573105.281672:0:2387:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573105.281679:0:2387:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc990 00000400:00000010:0.0:1788573105.281682:0:2387:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc990. 00000100:00000001:0.0:1788573105.281687:0:2387:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573105.281694:0:2387:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac820a80 x1875440516530560/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573105.281709:0:2387:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573105.281717:0:2387:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516530560/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573127 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573105.281730:0:2387:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573105.281733:0:2387:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573105.792916:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573105.792927:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573105.792931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573105.792940:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573105.792947:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573105.792954:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573105.792972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573105.792976:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573105.792981:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573106.048879:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308465901, at 4308465920 00000100:00000010:0.0:1788573106.048900:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7000. 00000100:00000001:0.0:1788573106.816929:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573106.816935:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573106.816937:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573106.816944:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573106.816951:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573106.816956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573106.816963:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573106.816966:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573106.816970:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573107.840931:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573107.840942:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573107.840948:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573107.840957:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573107.840965:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573107.840972:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573107.840991:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573107.840995:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573107.840999:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573108.545034:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573108.545038:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573108.545040:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573108.864934:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573108.864938:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573108.864944:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573108.864951:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573108.864956:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573108.864959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573108.864969:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573108.864972:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573108.864976:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573109.888926:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573109.888933:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573109.888935:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573109.888941:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573109.888946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573109.888951:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573109.888955:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573109.888959:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573109.888966:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573109.889020:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573109.889023:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573109.889025:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573109.889027:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573109.889029:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573109.889031:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573109.889033:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573109.889034:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573109.889045:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573110.912884:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573110.912890:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573110.912892:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573110.912899:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573110.912904:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573110.912909:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573110.912914:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573110.912919:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573110.912922:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573110.912949:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573110.912953:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573110.912954:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573110.912956:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573110.912958:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573110.912960:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573110.912962:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573110.912963:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573110.912965:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573111.168886:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308471022, at 4308471040 00000100:00000010:0.0:1788573111.168903:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7d80. 00000100:00000001:0.0:1788573111.936899:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573111.936907:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573111.936912:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573111.936920:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573111.936928:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573111.936935:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573111.936958:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573111.936961:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573111.936965:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573112.448895:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788573112.448904:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788573112.448911:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788573112.449075:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573112.449093:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573112.449101:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573112.449112:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573112.960914:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573112.960936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573112.960941:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573112.960960:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573112.960969:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573112.960975:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573112.960988:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573112.960992:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573112.960996:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573113.665063:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573113.665067:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573113.665070:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573113.984947:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573113.984958:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573113.984961:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573113.984972:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573113.984982:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573113.984987:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573113.985010:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573113.985012:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573113.985015:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573115.008904:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573115.008912:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573115.008916:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573115.008921:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573115.008928:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573115.008934:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573115.008947:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573115.008952:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573115.008956:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573116.032920:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573116.032927:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573116.032931:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573116.032937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573116.032944:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573116.032949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573116.032954:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573116.032975:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573116.032983:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573116.288943:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308476141, at 4308476160 00000100:00000010:0.0:1788573116.288961:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7840. 00000100:00000001:0.0:1788573117.056908:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573117.056919:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573117.056925:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573117.056933:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573117.056939:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573117.056944:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573117.056954:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573117.056957:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573117.056961:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573118.080957:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573118.080962:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573118.080966:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573118.080970:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573118.080975:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573118.080978:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573118.080981:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573118.080983:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573118.080985:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573118.785049:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573118.785052:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573118.785055:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573119.104873:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573119.104882:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573119.104885:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573119.104896:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573119.104903:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573119.104910:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573119.104925:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573119.104929:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573119.104932:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573120.128905:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573120.128916:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573120.128921:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573120.128944:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573120.128951:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573120.128957:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573120.128967:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573120.128971:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573120.128974:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573121.152942:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573121.152946:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573121.152950:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573121.152957:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573121.152963:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573121.152967:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573121.152972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573121.152973:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573121.152978:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573121.408892:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308481262, at 4308481280 00000100:00000010:0.0:1788573121.408908:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7720. 00000100:00000001:0.0:1788573122.176902:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573122.176908:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573122.176912:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573122.176920:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573122.176927:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573122.176933:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573122.176949:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573122.176952:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573122.176956:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573123.200940:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573123.200946:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573123.200950:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573123.200957:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573123.200963:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573123.200968:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573123.200972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573123.200974:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573123.200977:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573123.905032:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573123.905035:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573123.905062:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573124.224947:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573124.224955:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573124.224964:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573124.224982:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573124.224996:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573124.225009:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573124.225039:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573124.225043:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573124.225048:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573125.248964:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573125.248969:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573125.248972:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573125.248977:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573125.248982:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573125.248986:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573125.249008:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573125.249011:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573125.249015:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573126.272883:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573126.272891:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573126.272896:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573126.272905:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573126.272913:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573126.272920:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573126.272934:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573126.272937:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573126.272940:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573126.528906:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308486381, at 4308486400 00000100:00000010:0.0:1788573126.528922:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7900. 00000100:00000001:0.0:1788573127.296917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573127.296924:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573127.296930:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573127.296938:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573127.296944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573127.296949:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573127.296955:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573127.296973:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573127.296976:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573127.296988:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573127.296993:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573127.296994:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573127.296999:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573127.297002:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573127.297006:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573127.297023:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573127.297025:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573127.297029:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573128.320929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573128.320936:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573128.320941:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573128.320946:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573128.320951:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573128.320957:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573128.320960:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573128.320962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573128.320964:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573129.025034:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573129.025038:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573129.025047:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573129.344920:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573129.344928:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573129.344932:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573129.344949:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573129.344953:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573129.344957:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573129.344971:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573129.344973:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573129.344976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573130.368927:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573130.368932:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573130.368935:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573130.368941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573130.368948:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573130.368952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573130.368963:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573130.368966:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573130.368968:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573131.392929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573131.392939:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573131.392942:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573131.392951:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573131.392958:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573131.392964:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573131.392985:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573131.392986:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573131.392988:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573131.648923:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308491501, at 4308491520 00000100:00000010:0.0:1788573131.648939:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea74e0. 00000100:00000001:0.0:1788573132.416922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573132.416927:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573132.416930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573132.416936:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573132.416945:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573132.416949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573132.416970:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573132.416974:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573132.416977:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573134.145020:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573134.145024:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573134.145026:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573134.464927:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573134.464933:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573134.464937:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573134.464942:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573134.464947:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573134.464950:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573134.464963:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573134.464967:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573134.464971:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573135.488909:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573135.488915:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573135.488933:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573135.488937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573135.488942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573135.488948:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573135.488982:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573135.488986:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573135.488990:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573136.512883:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573136.512892:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573136.512906:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573136.512917:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573136.512923:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573136.512929:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573136.512941:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573136.512944:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573136.512949:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573136.768911:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308496621, at 4308496640 00000100:00000010:0.0:1788573136.768930:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7900. 00000100:00000001:0.0:1788573137.536906:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573137.536914:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573137.536917:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573137.536922:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573137.536928:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573137.536934:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573137.536942:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573137.536947:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573137.536951:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573138.560997:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573138.561006:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573138.561012:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573138.561018:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573138.561021:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573138.561026:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573138.561040:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573138.561049:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573138.561057:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573139.265065:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573139.265091:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573139.265093:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573139.584952:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573139.584966:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573139.584970:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573139.584975:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573139.584980:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573139.584985:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573139.584989:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573139.584993:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573139.584998:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573140.608930:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573140.608939:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573140.608943:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573140.608950:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573140.608955:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573140.608961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573140.608966:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573140.608969:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573140.608973:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573140.609009:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573140.609013:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573140.609016:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573140.609020:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573140.609024:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573140.609028:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573140.609031:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573140.609034:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573140.609037:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573141.632920:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573141.632926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573141.632930:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573141.632936:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573141.632942:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573141.632946:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573141.632951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573141.632956:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573141.632963:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573141.633000:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573141.633003:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573141.633005:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573141.633009:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573141.633013:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573141.633016:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573141.633017:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573141.633018:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573141.633020:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573141.888924:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308501742, at 4308501760 00000100:00000010:0.0:1788573141.888941:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916744e0. 00000100:00000001:0.0:1788573142.656971:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573142.656977:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573142.656980:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573142.656986:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573142.656993:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573142.656998:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573142.657002:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573142.657004:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573142.657009:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573143.680925:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573143.680932:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573143.680937:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573143.680944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573143.680950:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573143.680954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573143.680960:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573143.680962:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573143.680966:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573144.385002:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573144.385006:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573144.385010:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573144.704938:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573144.704947:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573144.704951:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573144.704959:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573144.704966:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573144.704991:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573144.704997:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573144.705004:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573144.705013:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573145.728936:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573145.728949:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573145.728957:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573145.728971:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573145.728983:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573145.728991:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573145.728998:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573145.729003:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573145.729011:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573146.752919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573146.752927:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573146.752932:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573146.752942:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573146.752951:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573146.752956:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573146.752963:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573146.752968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573146.752970:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573147.008881:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308506862, at 4308506880 00000100:00000010:0.0:1788573147.008903:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916745a0. 00000100:00000001:0.0:1788573147.776905:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573147.776914:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573147.776919:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573147.776927:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573147.776935:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573147.776942:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573147.776951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573147.776956:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573147.776964:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573148.288961:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573148.288972:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573148.288975:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573148.288994:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573148.289005:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573148.289011:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13835 00000400:00000200:0.0:1788573148.289019:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573148.289024:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573148.289026:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573148.289029:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289031:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573148.289033:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289035:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573148.289038:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573148.289040:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573148.289042:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573148.289044:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573148.289046:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573148.289047:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573148.289049:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289051:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573148.289052:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289053:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573148.289056:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573148.289058:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289060:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573148.289062:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573148.289065:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573148.289067:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.289068:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573148.289070:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573148.289075:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788573148.289078:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573148.289081:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573148.289088:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.289091:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573148.289093:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573148.289097:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573148.289104:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573148.289110:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573148.289116:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516535808/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573148.289125:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573148.289128:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573148.289132:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516535808/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573148.289148:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573148.289151:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573148.289153:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573148.289154:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573148.289156:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573148.289158:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573148.289159:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13835 00000400:00000200:0.0:1788573148.289162:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573148.289164:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573148.289166:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573148.289168:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289170:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573148.289172:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289174:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573148.289176:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573148.289177:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573148.289178:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573148.289180:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573148.289181:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573148.289187:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573148.289192:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289196:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573148.289201:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289203:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573148.289205:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573148.289210:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289215:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573148.289219:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573148.289227:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573148.289235:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.289241:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573148.289245:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573148.289252:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.289257:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573148.289263:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573148.289268:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573148.289281:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573148.289284:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.289286:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573148.289290:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573148.289295:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573148.289301:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573148.289307:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573148.289317:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516535936/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573148.289331:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573148.289335:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573148.289344:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516535936/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573148.289363:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573148.289374:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573148.289379:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573148.289380:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573148.289384:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573148.289390:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573148.289394:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13835 00000400:00000200:0.0:1788573148.289449:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573148.289458:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573148.289462:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573148.289467:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289471:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573148.289474:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289476:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573148.289480:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573148.289482:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573148.289483:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573148.289486:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573148.289489:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573148.289493:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573148.289500:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289504:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573148.289509:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289513:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573148.289516:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573148.289519:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289524:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573148.289528:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573148.289532:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573148.289535:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.289540:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573148.289549:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573148.289554:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.289556:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573148.289561:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573148.289565:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573148.289570:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573148.289572:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.289575:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573148.289578:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573148.289580:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573148.289586:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573148.289591:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573148.289603:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516536064/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573148.289623:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573148.289629:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573148.289637:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516536064/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573148.289673:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573148.289677:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573148.289681:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573148.289683:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573148.289687:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573148.289692:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573148.289697:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13835 00000400:00000200:0.0:1788573148.289703:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573148.289707:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573148.289709:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573148.289714:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289719:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573148.289723:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289733:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573148.289737:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573148.289740:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573148.289742:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573148.289744:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573148.289748:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573148.289751:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573148.289757:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289759:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573148.289764:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289768:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573148.289773:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573148.289780:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289786:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573148.289792:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573148.289798:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573148.289805:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.289836:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573148.289838:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573148.289844:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.289848:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573148.289855:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573148.289860:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573148.289864:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573148.289867:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.289870:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573148.289874:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573148.289877:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573148.289883:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573148.289891:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573148.289900:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516536192/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573148.289929:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573148.289934:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573148.289941:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516536192/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573148.289953:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573148.289957:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573148.289964:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573148.289967:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573148.289971:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573148.289974:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573148.289978:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13835 00000400:00000200:0.0:1788573148.289984:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573148.289990:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573148.289992:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573148.289994:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.289996:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573148.290001:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.290007:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573148.290009:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573148.290012:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573148.290017:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573148.290020:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573148.290023:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573148.290026:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573148.290030:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.290035:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573148.290037:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573148.290040:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573148.290048:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573148.290053:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573148.290059:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573148.290064:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573148.290068:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573148.290072:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.290075:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573148.290078:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573148.290080:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.290081:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573148.290088:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573148.290091:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573148.290097:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573148.290100:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573148.290105:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573148.290108:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573148.290111:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573148.290116:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573148.290121:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573148.290129:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516536320/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573148.290138:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573148.290142:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573148.290149:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516536320/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573148.290160:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573148.290167:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13845) 00000100:00000001:0.0:1788573148.800926:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573148.800932:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573148.800936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573148.800944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573148.800953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573148.800977:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573148.800984:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573148.800987:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573148.800991:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573149.505026:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573149.505029:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573149.505032:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573149.824916:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573149.824924:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573149.824929:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573149.824938:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573149.824947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573149.824954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573149.824980:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573149.824986:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573149.824993:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573150.848921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573150.848932:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573150.848937:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573150.848945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573150.848952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573150.848958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573150.848977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573150.848980:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573150.848983:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573151.360931:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788573151.360940:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788573151.360946:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788573151.361131:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573151.361145:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573151.361167:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573151.361170:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573151.872903:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573151.872912:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573151.872916:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573151.872923:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573151.872928:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573151.872965:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573151.872990:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573151.872993:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573151.872996:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573152.128901:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308511981, at 4308512000 00000100:00000010:0.0:1788573152.128911:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674a20. 00000100:00000001:0.0:1788573152.896907:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573152.896916:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573152.896920:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573152.896930:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573152.896937:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573152.896944:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573152.896953:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573152.896957:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573152.896959:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573153.920916:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573153.920922:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573153.920926:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573153.920944:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573153.920950:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573153.920956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573153.920979:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573153.920983:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573153.920988:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573154.625048:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573154.625052:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573154.625055:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573154.944988:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573154.944997:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573154.944999:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573154.945005:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573154.945021:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573154.945029:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573154.945038:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573154.945045:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573154.945051:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573155.968884:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573155.968907:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573155.968910:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573155.968917:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573155.968923:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573155.968928:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573155.968951:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573155.968955:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573155.968961:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573156.992918:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573156.992928:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573156.992934:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573156.992942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573156.992953:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573156.992961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573156.992965:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573156.992974:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573156.992983:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573157.248919:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308517102, at 4308517120 00000100:00000010:0.0:1788573157.248940:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674540. 00000100:00000001:0.0:1788573158.016933:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573158.016938:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573158.016941:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573158.016950:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573158.016968:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573158.016973:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573158.016979:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573158.016984:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573158.016991:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.017032:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573158.017036:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573158.017038:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573158.017040:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573158.017047:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573158.017049:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573158.017050:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573158.017051:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573158.017053:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573158.528967:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573158.528979:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573158.528983:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573158.528988:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573158.528994:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573158.529003:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13845 00000400:00000200:0.0:1788573158.529017:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573158.529028:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573158.529032:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573158.529035:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529038:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573158.529041:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529044:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573158.529048:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573158.529050:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573158.529053:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573158.529057:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573158.529059:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573158.529061:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573158.529064:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529065:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573158.529067:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529068:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573158.529070:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573158.529073:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529075:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573158.529077:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573158.529080:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573158.529083:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.529084:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573158.529086:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573158.529115:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573158.529119:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573158.529123:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573158.529124:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.529128:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573158.529131:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573158.529137:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573158.529145:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573158.529153:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573158.529162:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516537088/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573158.529172:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573158.529177:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573158.529181:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516537088/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573158.529200:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573158.529203:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573158.529206:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573158.529207:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573158.529209:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573158.529211:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573158.529213:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13845 00000400:00000200:0.0:1788573158.529230:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573158.529237:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573158.529246:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573158.529261:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529270:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573158.529275:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529277:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573158.529304:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573158.529307:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573158.529316:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573158.529320:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573158.529327:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573158.529331:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573158.529337:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529340:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573158.529344:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529347:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573158.529360:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573158.529370:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529372:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573158.529374:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573158.529380:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573158.529384:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.529389:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573158.529395:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573158.529405:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.529409:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573158.529420:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573158.529424:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573158.529426:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573158.529435:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.529438:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573158.529439:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573158.529442:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573158.529451:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573158.529457:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573158.529467:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516537216/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573158.529488:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573158.529494:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573158.529506:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516537216/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573158.529519:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573158.529527:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573158.529542:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573158.529549:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573158.529555:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573158.529559:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573158.529568:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13845 00000400:00000200:0.0:1788573158.529578:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573158.529590:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573158.529593:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573158.529602:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529605:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573158.529610:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529617:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573158.529621:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573158.529622:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573158.529623:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573158.529629:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573158.529636:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573158.529645:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573158.529666:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529678:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573158.529687:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529697:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573158.529706:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573158.529723:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573158.529731:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573158.529742:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573158.529750:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573158.529758:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.529762:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573158.529768:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573158.529775:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.529783:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573158.529789:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573158.529797:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573158.529802:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573158.529805:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.529845:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573158.529849:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573158.529851:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573158.529860:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573158.529870:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573158.529887:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516537344/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573158.529915:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573158.529920:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573158.529928:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516537344/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573158.529940:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573158.529945:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573158.529953:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573158.529957:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573158.529959:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573158.529968:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573158.529972:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13845 00000400:00000200:0.0:1788573158.529985:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573158.529991:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573158.529993:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573158.530011:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.530023:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573158.530033:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.530037:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573158.530043:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573158.530046:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573158.530050:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573158.530053:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573158.530065:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573158.530070:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573158.530075:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.530079:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573158.530083:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.530086:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573158.530096:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573158.530100:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573158.530102:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573158.530106:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573158.530114:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573158.530118:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.530121:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573158.530124:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573158.530127:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.530128:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573158.530131:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573158.530133:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573158.530136:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573158.530140:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.530143:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573158.530146:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573158.530149:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573158.530154:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573158.530163:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573158.530176:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516537472/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573158.530221:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573158.530231:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573158.530245:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516537472/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573158.530277:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573158.530295:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573158.530309:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573158.530316:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573158.530322:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573158.530329:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573158.530334:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13845 00000400:00000200:0.0:1788573158.530344:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573158.530349:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573158.530355:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573158.530362:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.530366:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573158.530369:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.530370:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573158.530373:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573158.530376:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573158.530380:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573158.530383:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573158.530387:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573158.530392:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573158.530401:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.530404:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573158.530414:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573158.530417:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573158.530421:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573158.530430:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573158.530440:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573158.530450:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573158.530455:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573158.530460:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.530462:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573158.530465:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573158.530472:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.530475:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573158.530484:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573158.530489:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573158.530497:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573158.530507:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573158.530520:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573158.530526:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573158.530529:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573158.530540:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573158.530548:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573158.530567:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516537600/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573158.530597:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573158.530600:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573158.530610:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516537600/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573158.530666:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573158.530679:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13856) 00000100:00000001:0.0:1788573159.040959:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573159.040967:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573159.040972:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573159.041001:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573159.041017:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573159.041025:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573159.041031:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573159.041035:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573159.041040:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573159.745054:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573159.745060:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573159.745064:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573160.064944:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573160.064962:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573160.064968:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573160.064983:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573160.064994:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573160.065002:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573160.065026:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573160.065031:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573160.065036:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573161.088921:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573161.088927:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573161.088930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573161.088936:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573161.088941:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573161.088944:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573161.088956:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573161.088962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573161.088969:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573162.112907:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573162.112917:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573162.112923:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573162.112931:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573162.112940:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573162.112944:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573162.112957:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573162.112959:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573162.112966:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573162.368881:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308522222, at 4308522240 00000100:00000010:0.0:1788573162.368897:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674c60. 00000100:00000001:0.0:1788573163.136922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573163.136929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573163.136932:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573163.136938:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573163.136945:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573163.136949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.136963:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573163.136966:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.136969:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.137005:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573163.137008:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573163.137010:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573163.137012:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573163.137015:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573163.137017:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.137019:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573163.137020:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573163.137023:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573163.648922:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573163.648933:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573163.648936:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573163.648939:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573163.648944:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573163.648950:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13851 00000400:00000200:0.0:1788573163.648959:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573163.648966:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573163.648968:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573163.648972:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.648975:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573163.648977:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.648980:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573163.648983:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573163.648984:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573163.648987:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573163.649005:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573163.649007:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573163.649009:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573163.649011:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649013:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573163.649015:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649016:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573163.649018:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573163.649021:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649023:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.649025:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573163.649028:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573163.649030:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.649031:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573163.649033:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573163.649038:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573163.649041:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573163.649044:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573163.649045:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.649048:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573163.649050:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573163.649054:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573163.649069:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573163.649075:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573163.649086:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516537728/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573163.649094:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573163.649098:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573163.649133:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516537728/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.649151:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573163.649154:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573163.649163:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573163.649164:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573163.649166:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573163.649168:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573163.649170:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13851 00000400:00000200:0.0:1788573163.649175:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573163.649178:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573163.649179:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573163.649183:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649187:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573163.649190:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649194:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573163.649198:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573163.649199:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573163.649201:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573163.649204:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573163.649208:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573163.649213:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573163.649220:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649224:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573163.649229:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649233:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573163.649237:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573163.649243:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649249:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.649253:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573163.649261:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573163.649267:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.649273:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573163.649280:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573163.649292:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.649297:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573163.649303:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573163.649309:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573163.649314:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573163.649316:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.649318:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573163.649321:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573163.649325:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573163.649332:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573163.649338:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573163.649346:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516537856/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573163.649358:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573163.649362:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573163.649370:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516537856/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.649380:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573163.649385:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573163.649388:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573163.649391:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573163.649394:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573163.649398:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573163.649403:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13851 00000400:00000200:0.0:1788573163.649440:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573163.649447:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573163.649450:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573163.649453:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649455:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573163.649459:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649468:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573163.649474:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573163.649476:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573163.649479:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573163.649483:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573163.649487:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573163.649492:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573163.649497:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649501:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573163.649503:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649504:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573163.649506:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573163.649508:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649512:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.649516:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573163.649519:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573163.649523:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.649525:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573163.649528:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573163.649532:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.649536:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573163.649540:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573163.649546:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573163.649550:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573163.649552:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.649554:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573163.649558:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573163.649560:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573163.649564:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573163.649566:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573163.649578:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516537984/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573163.649615:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573163.649620:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573163.649630:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516537984/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.649647:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573163.649651:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573163.649655:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573163.649656:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573163.649660:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573163.649662:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573163.649665:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13851 00000400:00000200:0.0:1788573163.649673:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573163.649678:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573163.649681:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573163.649694:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649699:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573163.649707:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649712:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573163.649715:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573163.649718:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573163.649723:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573163.649726:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573163.649731:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573163.649736:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573163.649743:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649748:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573163.649753:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649758:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573163.649770:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573163.649780:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649787:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.649793:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573163.649798:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573163.649801:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.649802:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573163.649805:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573163.649826:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.649828:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573163.649833:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573163.649836:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573163.649839:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573163.649842:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.649846:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573163.649850:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573163.649855:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573163.649864:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573163.649870:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573163.649881:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516538112/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573163.649894:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573163.649899:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573163.649906:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516538112/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.649918:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573163.649921:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573163.649927:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573163.649930:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573163.649934:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573163.649937:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573163.649948:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13851 00000400:00000200:0.0:1788573163.649958:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573163.649967:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573163.649973:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573163.649982:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.649988:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573163.649994:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.650001:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573163.650008:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573163.650012:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573163.650016:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573163.650019:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573163.650024:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573163.650030:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573163.650038:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.650042:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573163.650045:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573163.650047:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573163.650049:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573163.650054:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573163.650056:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573163.650058:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573163.650061:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573163.650066:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.650068:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573163.650071:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573163.650075:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.650077:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573163.650083:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573163.650088:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573163.650093:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573163.650114:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573163.650121:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573163.650125:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573163.650131:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573163.650137:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573163.650143:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573163.650151:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516538240/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573163.650162:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573163.650167:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573163.650173:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516538240/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573163.650187:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573163.650193:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13861) 00000100:00000001:0.0:1788573164.160955:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573164.160960:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573164.160964:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573164.160968:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573164.160973:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573164.160977:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573164.160980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573164.160981:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573164.160983:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573164.865029:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573164.865034:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573164.865038:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573165.184944:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573165.184951:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573165.184955:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573165.184962:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573165.184968:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573165.184973:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573165.184978:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573165.184982:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573165.184987:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573166.208953:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573166.208960:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573166.208965:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573166.208972:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573166.208979:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573166.208985:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573166.208989:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573166.208991:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573166.208995:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573167.233159:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573167.233169:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573167.233175:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573167.233189:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573167.233202:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573167.233212:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573167.233226:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573167.233233:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573167.233238:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573167.488888:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308527341, at 4308527360 00000100:00000010:0.0:1788573167.488908:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674540. 00000100:00000001:0.0:1788573168.256964:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573168.256969:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573168.256973:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573168.256978:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573168.256982:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573168.256986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573168.256989:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573168.256991:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573168.256994:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573169.985013:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573169.985016:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573169.985019:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573170.304964:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573170.304969:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573170.304972:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573170.304975:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573170.304982:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573170.305001:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573170.305011:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573170.305014:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573170.305020:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573171.328897:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573171.328907:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573171.328911:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573171.328921:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573171.328927:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573171.328931:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573171.328935:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573171.328939:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573171.328943:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788573171.841079:0:2386:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788573171.841120:0:2386:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788573171.841127:0:2386:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573171.841130:0:2386:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788573171.841135:0:2386:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573171.841138:0:2386:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788573171.841142:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573171.841145:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793600. 00000400:00000200:0.0:1788573171.841150:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573171.841153:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573171.841159:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc198 00000400:00000010:0.0:1788573171.841162:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc198. 00000100:00000001:0.0:1788573171.841167:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573171.841177:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f4700 x1875440516538368/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573171.841204:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573171.841209:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4700 x1875440516538368/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573171.841217:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573171.841219:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573171.841231:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573171.841252:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793300. 00000400:00000200:0.0:1788573171.841257:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573171.841265:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573171.841273:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc7f8 00000400:00000010:0.0:1788573171.841276:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc7f8. 00000100:00000001:0.0:1788573171.841282:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573171.841288:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f6680 x1875440516538496/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573171.841306:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573171.841312:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6680 x1875440516538496/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573171.841324:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573171.841328:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573171.841339:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573171.841344:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793800. 00000400:00000200:0.0:1788573171.841350:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573171.841360:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573171.841367:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcee0 00000400:00000010:0.0:1788573171.841373:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcee0. 00000100:00000001:0.0:1788573171.841378:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573171.841394:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f5f80 x1875440516538624/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573171.841406:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573171.841411:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5f80 x1875440516538624/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573171.841426:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573171.841428:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573171.841434:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573171.841437:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793a00. 00000400:00000200:0.0:1788573171.841440:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573171.841447:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573171.841461:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc2a8 00000400:00000010:0.0:1788573171.841467:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc2a8. 00000100:00000001:0.0:1788573171.841471:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573171.841478:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f7800 x1875440516538752/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573171.841494:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573171.841501:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7800 x1875440516538752/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573171.841513:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573171.841518:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573171.841529:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573171.841533:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793000. 00000400:00000200:0.0:1788573171.841537:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573171.841544:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573171.841548:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc000 00000400:00000010:0.0:1788573171.841552:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc000. 00000100:00000001:0.0:1788573171.841557:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573171.841565:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f7480 x1875440516538880/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573171.841578:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573171.841581:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7480 x1875440516538880/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573193 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573171.841588:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573171.841592:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573172.352991:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573172.352995:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573172.352998:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573172.353002:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573172.353005:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573172.353008:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573172.353011:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573172.353015:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573172.353020:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573172.608921:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308532461, at 4308532480 00000100:00000010:0.0:1788573172.608938:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea71e0. 00000100:00000001:0.0:1788573173.377045:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573173.377050:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573173.377054:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573173.377058:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573173.377062:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573173.377065:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573173.377086:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573173.377098:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573173.377106:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573174.400980:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573174.400991:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573174.400998:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573174.401016:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573174.401034:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573174.401041:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573174.401070:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573174.401074:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573174.401078:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573175.105104:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573175.105109:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573175.105114:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573175.424918:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573175.424931:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573175.424934:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573175.424945:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573175.424959:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573175.424966:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573175.424973:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573175.424978:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573175.424987:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573175.425054:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573175.425057:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573175.425059:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573175.425062:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573175.425067:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573175.425083:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573175.425085:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573175.425086:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573175.425090:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573176.448937:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573176.448946:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573176.448952:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573176.448963:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573176.448971:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573176.448978:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573176.448998:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573176.449004:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573176.449017:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573176.449040:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573176.449046:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573176.449050:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573176.449054:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573176.449059:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573176.449062:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573176.449066:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573176.449068:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573176.449071:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573177.472954:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573177.472961:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573177.472966:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573177.472972:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573177.472978:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573177.472982:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573177.472987:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573177.472991:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573177.472998:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573177.728963:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308537582, at 4308537600 00000100:00000010:0.0:1788573177.728976:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f00. 00000100:00000001:0.0:1788573178.496976:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573178.496996:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573178.497005:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573178.497022:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573178.497033:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573178.497055:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573178.497091:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573178.497102:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573178.497111:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.520902:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573179.520908:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573179.520911:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573179.520917:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573179.520924:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573179.520927:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573179.520942:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573179.520947:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573179.520951:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573179.520975:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573179.520979:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573179.520981:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573179.520985:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573179.520988:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573179.520991:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573179.520993:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573179.520995:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573179.520998:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573180.225000:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573180.225003:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573180.225006:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573180.544901:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573180.544909:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573180.544914:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573180.544923:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573180.544929:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573180.544933:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573180.544939:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573180.544944:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573180.544949:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573180.544975:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573180.544979:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573180.544982:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573180.544986:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573180.544995:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573180.544998:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573180.544999:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573180.545001:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573180.545002:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573181.569011:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573181.569016:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573181.569018:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573181.569021:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573181.569024:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573181.569026:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573181.569028:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573181.569031:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573181.569034:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573182.592973:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573182.592980:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573182.592985:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573182.592992:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573182.592999:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573182.593005:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573182.593008:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573182.593010:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573182.593013:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573182.848913:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308542702, at 4308542720 00000100:00000010:0.0:1788573182.848930:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7d80. 00000100:00000001:0.0:1788573183.616931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573183.616937:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573183.616942:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573183.616947:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573183.616953:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573183.616958:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573183.616977:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573183.616980:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573183.616982:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573184.128916:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573184.128926:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573184.128944:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573184.128947:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573184.128950:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573184.128956:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13871 00000400:00000200:0.0:1788573184.128963:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573184.128969:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573184.128971:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573184.128974:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.128976:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573184.128978:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.128980:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573184.128983:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573184.128984:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573184.128986:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573184.128988:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573184.128990:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573184.128992:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573184.128994:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.128995:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573184.128997:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.128998:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573184.129000:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573184.129002:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129004:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573184.129006:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573184.129008:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573184.129010:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129012:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573184.129013:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573184.129018:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573184.129021:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573184.129027:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573184.129029:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129031:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573184.129033:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573184.129037:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573184.129043:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573184.129048:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573184.129055:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516540288/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573184.129063:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573184.129067:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573184.129071:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516540288/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573184.129089:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573184.129098:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573184.129101:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573184.129102:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573184.129103:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573184.129105:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573184.129108:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13871 00000400:00000200:0.0:1788573184.129111:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573184.129114:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573184.129116:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573184.129119:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129122:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573184.129125:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129127:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573184.129131:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573184.129133:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573184.129136:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573184.129139:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573184.129145:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573184.129149:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573184.129159:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129163:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573184.129165:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129167:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573184.129168:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573184.129170:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129172:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573184.129174:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573184.129181:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573184.129184:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129188:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573184.129194:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573184.129198:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129202:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573184.129207:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573184.129209:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573184.129213:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573184.129214:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129219:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573184.129222:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573184.129224:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573184.129226:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573184.129229:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573184.129236:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516540416/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573184.129245:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573184.129248:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573184.129251:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516540416/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573184.129277:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573184.129280:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573184.129285:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573184.129290:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573184.129292:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573184.129296:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573184.129302:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13871 00000400:00000200:0.0:1788573184.129310:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573184.129315:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573184.129319:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573184.129326:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129331:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573184.129333:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129336:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573184.129339:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573184.129340:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573184.129342:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573184.129347:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573184.129350:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573184.129355:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573184.129362:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129368:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573184.129373:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129379:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573184.129384:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573184.129404:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129409:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573184.129418:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573184.129422:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573184.129425:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129429:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573184.129433:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573184.129439:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129442:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573184.129446:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573184.129452:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573184.129459:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573184.129462:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129465:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573184.129468:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573184.129472:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573184.129481:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573184.129486:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573184.129494:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516540544/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573184.129508:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573184.129510:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573184.129517:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516540544/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573184.129528:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573184.129532:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573184.129535:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573184.129538:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573184.129540:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573184.129543:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573184.129549:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13871 00000400:00000200:0.0:1788573184.129555:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573184.129561:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573184.129564:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573184.129574:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129579:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573184.129581:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129584:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573184.129588:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573184.129590:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573184.129591:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573184.129594:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573184.129596:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573184.129600:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573184.129602:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129604:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573184.129605:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129606:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573184.129608:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573184.129611:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129615:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573184.129616:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573184.129619:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573184.129623:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129625:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573184.129627:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573184.129629:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129631:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573184.129636:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573184.129639:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573184.129641:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573184.129643:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129646:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573184.129647:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573184.129649:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573184.129655:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573184.129663:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573184.129669:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516540672/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573184.129678:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573184.129681:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573184.129689:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516540672/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573184.129698:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573184.129702:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573184.129707:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573184.129708:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573184.129711:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573184.129714:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573184.129717:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13871 00000400:00000200:0.0:1788573184.129724:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573184.129729:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573184.129733:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573184.129738:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129741:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573184.129746:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129751:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573184.129757:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573184.129761:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573184.129766:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573184.129769:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573184.129774:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573184.129781:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573184.129790:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129794:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573184.129803:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129826:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573184.129829:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573184.129833:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573184.129837:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573184.129841:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573184.129845:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573184.129848:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129851:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573184.129852:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573184.129856:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129859:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573184.129867:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573184.129872:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573184.129876:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573184.129879:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573184.129882:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573184.129885:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573184.129887:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573184.129892:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573184.129898:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573184.129906:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516540800/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573184.129919:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573184.129922:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573184.129929:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516540800/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573184.129946:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573184.129951:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13881) 00000100:00000001:0.0:1788573184.640902:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573184.640907:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573184.640913:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573184.640934:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573184.640941:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573184.640946:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573184.640950:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573184.640953:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573184.640957:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573185.345033:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573185.345036:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573185.345039:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573185.664932:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573185.664942:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573185.664945:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573185.664952:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573185.664960:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573185.664965:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.664976:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573185.664980:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573185.664984:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573186.688889:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573186.688897:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573186.688900:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573186.688906:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573186.688913:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573186.688918:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573186.688924:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573186.688930:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573186.688938:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573186.688972:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573186.688976:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573186.688977:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573186.688980:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573186.688983:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573186.688985:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573186.688987:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573186.688988:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573186.688990:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573187.712952:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573187.712959:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573187.712979:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573187.712985:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573187.712991:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573187.712995:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573187.713008:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573187.713013:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573187.713021:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573187.968917:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308547821, at 4308547840 00000100:00000010:0.0:1788573187.968936:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c00. 00000100:00000001:0.0:1788573188.736912:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573188.736920:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573188.736925:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573188.736931:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573188.736935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573188.736940:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573188.736950:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573188.736953:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573188.736956:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573189.248940:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788573189.248952:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788573189.248969:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1788573189.249171:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788573189.249178:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573189.249195:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573189.249201:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573189.249222:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573189.760947:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573189.760953:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573189.760958:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573189.760966:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573189.760975:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573189.760981:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573189.760997:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573189.761000:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573189.761002:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573190.465044:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573190.465058:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573190.465061:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573190.784919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573190.784924:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573190.784929:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573190.784934:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573190.784939:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573190.784945:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.784952:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573190.784955:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573190.784957:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573191.808912:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573191.808922:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573191.808927:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573191.808936:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573191.808945:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573191.808950:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573191.808998:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573191.809001:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573191.809005:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573192.832929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573192.832939:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573192.832943:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573192.832954:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573192.832964:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573192.832971:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573192.832977:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573192.832981:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573192.832987:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573193.088902:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308552941, at 4308552960 00000100:00000010:0.0:1788573193.088924:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ba0. 00000100:00000001:0.0:1788573193.856961:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573193.856966:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573193.856969:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573193.856976:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573193.856980:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573193.856984:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573193.857003:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573193.857007:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573193.857013:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573194.368929:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573194.368939:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573194.368942:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573194.368946:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573194.368950:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573194.368957:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13881 00000400:00000200:0.0:1788573194.368964:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573194.368971:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573194.368974:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573194.368976:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.368978:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573194.368980:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.368982:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573194.368985:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573194.368987:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573194.368989:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573194.368991:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573194.368992:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573194.368994:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573194.368996:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.368997:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573194.368999:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369000:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573194.369002:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573194.369005:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369006:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573194.369008:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573194.369011:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573194.369055:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.369058:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573194.369059:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573194.369065:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788573194.369068:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573194.369071:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573194.369073:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.369075:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573194.369077:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573194.369081:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573194.369087:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573194.369093:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573194.369101:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0e00 x1875440516541568/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573194.369109:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573194.369112:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573194.369115:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0e00 x1875440516541568/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573194.369132:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573194.369135:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573194.369137:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573194.369138:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573194.369140:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573194.369142:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573194.369144:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13881 00000400:00000200:0.0:1788573194.369148:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573194.369151:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573194.369153:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573194.369158:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369162:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573194.369165:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369175:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573194.369180:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573194.369183:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573194.369186:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573194.369189:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573194.369192:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573194.369197:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573194.369205:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369209:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573194.369214:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369218:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573194.369221:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573194.369224:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369225:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573194.369226:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573194.369232:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573194.369235:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.369238:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573194.369241:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573194.369248:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.369250:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573194.369255:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573194.369259:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573194.369263:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573194.369264:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.369266:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573194.369270:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573194.369272:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573194.369276:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573194.369283:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573194.369291:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516541696/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573194.369331:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573194.369335:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573194.369344:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516541696/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573194.369361:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573194.369366:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573194.369372:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573194.369379:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573194.369384:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573194.369393:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573194.369400:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13881 00000400:00000200:0.0:1788573194.369421:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573194.369428:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573194.369431:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573194.369436:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369439:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573194.369444:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369446:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573194.369452:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573194.369454:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573194.369457:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573194.369461:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573194.369466:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573194.369470:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573194.369473:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369475:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573194.369478:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369480:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573194.369489:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573194.369494:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369496:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573194.369497:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573194.369501:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573194.369505:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.369510:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573194.369514:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573194.369522:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.369526:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573194.369534:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573194.369540:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573194.369547:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573194.369551:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.369557:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573194.369561:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573194.369567:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573194.369575:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573194.369583:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573194.369594:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516541824/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573194.369608:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573194.369612:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573194.369620:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516541824/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573194.369634:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573194.369636:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573194.369644:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573194.369647:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573194.369650:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573194.369654:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573194.369666:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13881 00000400:00000200:0.0:1788573194.369674:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573194.369678:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573194.369680:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573194.369686:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369692:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573194.369697:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369703:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573194.369707:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573194.369708:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573194.369711:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573194.369714:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573194.369717:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573194.369721:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573194.369729:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369731:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573194.369737:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369743:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573194.369745:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573194.369749:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369754:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573194.369756:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573194.369760:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573194.369765:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.369770:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573194.369773:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573194.369780:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.369783:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573194.369789:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573194.369795:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573194.369803:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573194.369837:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.369844:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573194.369849:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573194.369854:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573194.369861:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573194.369868:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573194.369875:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516541952/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573194.369895:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573194.369899:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573194.369904:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516541952/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573194.369919:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573194.369922:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573194.369927:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573194.369930:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573194.369933:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573194.369936:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573194.369941:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13881 00000400:00000200:0.0:1788573194.369946:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573194.369950:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573194.369952:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573194.369954:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369957:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573194.369962:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.369965:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573194.369968:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573194.369973:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573194.369977:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573194.369979:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573194.369983:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573194.369993:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573194.370000:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.370005:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573194.370010:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573194.370016:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573194.370018:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573194.370022:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573194.370026:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573194.370042:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573194.370046:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573194.370051:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.370055:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573194.370057:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573194.370061:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.370065:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573194.370068:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573194.370074:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573194.370079:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573194.370082:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573194.370087:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573194.370088:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573194.370092:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573194.370096:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573194.370102:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573194.370111:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516542080/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573194.370126:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573194.370128:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573194.370134:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516542080/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573194.370163:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573194.370175:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13891) 00000100:00000001:0.0:1788573194.880920:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573194.880927:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573194.880931:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573194.880937:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573194.880942:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573194.880945:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573194.880949:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573194.880952:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573194.880957:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573195.585053:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573195.585058:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573195.585061:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573195.904943:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573195.904959:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573195.904971:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573195.904996:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573195.905004:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573195.905011:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.905024:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573195.905033:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573195.905050:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573196.928891:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573196.928896:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573196.928900:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573196.928905:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573196.928909:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573196.928913:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573196.928931:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573196.928935:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573196.928939:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573197.952961:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573197.952972:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573197.952977:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573197.952986:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573197.952993:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573197.952997:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573197.953000:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573197.953013:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573197.953018:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573198.208929:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308558061, at 4308558080 00000100:00000010:0.0:1788573198.208948:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7600. 00000100:00000001:0.0:1788573198.976903:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573198.976912:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573198.976916:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573198.976924:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573198.976932:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573198.976938:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573198.976945:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573198.976951:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573198.976958:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573198.976995:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573198.977001:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573198.977004:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573198.977007:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573198.977009:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573198.977011:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573198.977013:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573198.977014:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573198.977016:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.000911:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573200.000919:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573200.000921:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573200.000927:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573200.000934:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573200.000938:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.000943:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573200.000946:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.000950:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573200.000969:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573200.000974:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573200.000979:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573200.000985:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573200.000991:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573200.000995:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.001013:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573200.001014:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573200.001016:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573200.705039:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573200.705044:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573200.705048:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573201.024910:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573201.024916:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573201.024918:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573201.024922:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573201.024926:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573201.024929:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.024933:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573201.024938:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.024946:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573201.024981:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573201.024986:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573201.024988:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573201.024990:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573201.024995:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573201.024998:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.025001:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573201.025004:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573201.025007:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573202.048955:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573202.048966:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573202.048970:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573202.048974:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573202.048980:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573202.048982:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573202.048985:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573202.048987:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573202.048990:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573203.072940:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573203.072951:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573203.072956:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573203.072966:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573203.072974:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573203.072995:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573203.073012:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573203.073015:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573203.073018:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573203.328909:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308563181, at 4308563200 00000100:00000010:0.0:1788573203.328920:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7b40. 00000100:00000001:0.0:1788573204.096972:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573204.096977:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573204.096980:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573204.096986:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573204.096993:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573204.096997:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573204.097000:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573204.097003:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573204.097008:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573204.608924:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573204.608934:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573204.608937:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573204.608941:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573204.608946:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573204.608954:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13892 00000400:00000200:0.0:1788573204.608963:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573204.608969:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573204.608972:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573204.608975:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.608977:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573204.608980:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.608982:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573204.608984:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573204.608986:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573204.608989:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573204.608991:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573204.608992:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573204.609011:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573204.609014:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609015:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573204.609017:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609018:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573204.609020:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573204.609022:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609024:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573204.609026:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573204.609029:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573204.609031:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609033:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573204.609034:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573204.609039:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573204.609042:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573204.609044:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573204.609046:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609048:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573204.609050:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573204.609054:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573204.609060:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573204.609065:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573204.609072:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516542848/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573204.609080:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573204.609083:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573204.609087:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516542848/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573204.609103:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573204.609106:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573204.609108:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573204.609117:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573204.609119:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573204.609123:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573204.609127:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13892 00000400:00000200:0.0:1788573204.609132:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573204.609140:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573204.609146:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573204.609153:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609159:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573204.609164:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609168:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573204.609172:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573204.609174:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573204.609177:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573204.609179:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573204.609183:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573204.609185:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573204.609190:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609193:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573204.609197:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609199:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573204.609202:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573204.609208:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609212:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573204.609216:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573204.609221:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573204.609227:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609231:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573204.609236:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573204.609240:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609249:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573204.609255:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573204.609258:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573204.609261:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573204.609263:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609265:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573204.609266:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573204.609268:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573204.609275:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573204.609280:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573204.609290:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516542976/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573204.609304:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573204.609307:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573204.609315:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516542976/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573204.609324:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573204.609326:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573204.609330:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573204.609331:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573204.609332:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573204.609335:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573204.609339:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13892 00000400:00000200:0.0:1788573204.609345:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573204.609354:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573204.609359:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573204.609375:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609380:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573204.609387:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609392:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573204.609397:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573204.609405:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573204.609407:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573204.609410:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573204.609414:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573204.609418:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573204.609425:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609428:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573204.609434:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609439:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573204.609444:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573204.609453:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609459:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573204.609465:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573204.609471:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573204.609478:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609480:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573204.609482:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573204.609490:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609496:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573204.609501:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573204.609504:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573204.609508:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573204.609509:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609511:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573204.609512:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573204.609516:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573204.609523:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573204.609527:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573204.609534:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516543104/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573204.609548:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573204.609551:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573204.609564:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516543104/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573204.609576:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573204.609580:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573204.609585:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573204.609588:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573204.609591:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573204.609595:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573204.609599:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13892 00000400:00000200:0.0:1788573204.609605:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573204.609610:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573204.609616:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573204.609620:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609626:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573204.609629:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609634:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573204.609640:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573204.609642:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573204.609646:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573204.609650:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573204.609654:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573204.609657:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573204.609662:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609663:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573204.609668:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609672:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573204.609678:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573204.609681:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609686:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573204.609695:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573204.609699:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573204.609703:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609707:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573204.609711:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573204.609715:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609720:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573204.609726:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573204.609729:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573204.609733:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573204.609735:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609738:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573204.609740:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573204.609743:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573204.609749:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573204.609753:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573204.609778:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516543232/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573204.609790:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573204.609792:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573204.609801:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516543232/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573204.609844:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573204.609850:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573204.609858:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573204.609861:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573204.609864:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573204.609871:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573204.609876:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13892 00000400:00000200:0.0:1788573204.609883:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573204.609890:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573204.609895:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573204.609906:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609910:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573204.609915:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609918:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573204.609921:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573204.609924:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573204.609925:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573204.609927:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573204.609931:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573204.609935:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573204.609942:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609946:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573204.609949:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609950:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573204.609952:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573204.609957:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573204.609960:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573204.609963:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573204.609966:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573204.609968:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609970:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573204.609973:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573204.609976:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.609980:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573204.609986:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573204.609989:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573204.609994:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573204.609997:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573204.610001:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573204.610003:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573204.610008:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573204.610016:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573204.610026:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573204.610034:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516543360/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573204.610046:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573204.610050:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573204.610056:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516543360/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573204.610072:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573204.610078:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13902) 00000020:00000001:0.0:1788573204.801075:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573204.801079:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573204.801084:0:241699:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 7 locks of 0 total in blp. (0 blwis in pool) 00010000:00000001:0.0:1788573204.801086:0:241699:0:(ldlm_lockd.c:2881:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1788573204.801091:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573204.801099:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b81e5400/0x7d52aadf1286ce3c lrc: 2/0,0 mode: PR/PR res: [0x2400013a6:0x10fa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x784a792ceaa5b3d4 expref: -99 pid: 265055 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00000001:0.0:1788573204.801121:0:241699:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788573204.801129:0:241699:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788573204.801134:0:241699:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788573204.801139:0:241699:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x2400013a6:0x10fa:0x0] mode (null) 00000002:00000001:0.0:1788573204.801149:0:241699:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788573204.801155:0:241699:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788573204.801163:0:241699:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff8fc4a64d9e00 count: 2 00010000:00000001:0.0:1788573204.801167:0:241699:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788573204.801176:0:241699:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.801181:0:241699:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.801185:0:241699:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:0.0:1788573204.801190:0:241699:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x2400013a6:0x10fa:0x0](ffff8fc498f8b248) invalid 00000020:00000001:0.0:1788573204.801196:0:241699:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffff9d48c1483ca0 00000080:00002000:0.0:1788573204.801200:0:241699:0:(llite_internal.h:2029:d_lustre_invalidate()) invalidate dentry d4 (ffff8fc498f1daf8) parent ffff8fc498f1d5e8 inode ffff8fc498f8b248 refc 2 00000080:00000001:0.0:1788573204.801207:0:241699:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:0.0:1788573204.801211:0:241699:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788573204.801223:0:241699:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573204.801228:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573204.801234:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573204.801240:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573204.801243:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573204.801248:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573204.801252:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b81e5400 with handle 0x7d52aadf1286ce3c from hash 00010000:00000001:0.0:1788573204.801258:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573204.801261:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573204.801265:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.801269:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573204.801273:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573204.801278:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573204.801282:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573204.801286:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b81e4200/0x7d52aadf1286ce20 lrc: 2/0,0 mode: PR/PR res: [0x2400013a6:0x10f6:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x784a792ceaa5b380 expref: -99 pid: 265051 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00000001:0.0:1788573204.801302:0:241699:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788573204.801306:0:241699:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788573204.801308:0:241699:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788573204.801310:0:241699:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x2400013a6:0x10f6:0x0] mode (null) 00000002:00000001:0.0:1788573204.801315:0:241699:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788573204.801319:0:241699:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788573204.801323:0:241699:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff8fc489e3ee00 count: 2 00010000:00000001:0.0:1788573204.801327:0:241699:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788573204.801334:0:241699:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.801339:0:241699:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.801344:0:241699:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:0.0:1788573204.801348:0:241699:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x2400013a6:0x10f6:0x0](ffff8fc491dbcb08) invalid 00000020:00000001:0.0:1788573204.801354:0:241699:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffff9d48c1483ca0 00000080:00002000:0.0:1788573204.801357:0:241699:0:(llite_internal.h:2029:d_lustre_invalidate()) invalidate dentry d0 (ffff8fc498f1d6c0) parent ffff8fc498f1d5e8 inode ffff8fc491dbcb08 refc 2 00000080:00000001:0.0:1788573204.801362:0:241699:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:0.0:1788573204.801364:0:241699:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788573204.801365:0:241699:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573204.801366:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573204.801367:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573204.801375:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573204.801380:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573204.801384:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573204.801387:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b81e4200 with handle 0x7d52aadf1286ce20 from hash 00010000:00000001:0.0:1788573204.801393:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573204.801395:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573204.801399:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.801402:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573204.801407:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573204.801410:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573204.801414:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573204.801418:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4a635d000/0x7d52aadf1286ce35 lrc: 2/0,0 mode: PR/PR res: [0x2400013a6:0x10fb:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x784a792ceaa5b3db expref: -99 pid: 265056 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00000001:0.0:1788573204.801426:0:241699:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788573204.801430:0:241699:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788573204.801432:0:241699:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788573204.801435:0:241699:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x2400013a6:0x10fb:0x0] mode (null) 00000002:00000001:0.0:1788573204.801439:0:241699:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788573204.801443:0:241699:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788573204.801447:0:241699:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff8fc489e3e600 count: 2 00010000:00000001:0.0:1788573204.801452:0:241699:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788573204.801457:0:241699:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.801462:0:241699:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.801465:0:241699:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:0.0:1788573204.801467:0:241699:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x2400013a6:0x10fb:0x0](ffff8fc498f880c8) invalid 00000020:00000001:0.0:1788573204.801470:0:241699:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffff9d48c1483ca0 00000080:00002000:0.0:1788573204.801473:0:241699:0:(llite_internal.h:2029:d_lustre_invalidate()) invalidate dentry d5 (ffff8fc491ed8a20) parent ffff8fc498f1d5e8 inode ffff8fc498f880c8 refc 0 00000080:00000001:0.0:1788573204.801505:0:241699:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:0.0:1788573204.801514:0:241699:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:0.0:1788573204.801520:0:241699:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:0.0:1788573204.801526:0:241699:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788573204.801531:0:241699:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573204.801535:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573204.801539:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573204.801543:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573204.801551:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573204.801555:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573204.801561:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4a635d000 with handle 0x7d52aadf1286ce35 from hash 00010000:00000001:0.0:1788573204.801566:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573204.801570:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573204.801573:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.801578:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573204.801581:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573204.801586:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573204.801591:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573204.801601:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b81e5a00/0x7d52aadf1286ce27 lrc: 2/0,0 mode: PR/PR res: [0x2400013a6:0x10f7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x784a792ceaa5b3a3 expref: -99 pid: 265052 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00000001:0.0:1788573204.801622:0:241699:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788573204.801629:0:241699:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788573204.801632:0:241699:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788573204.801634:0:241699:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x2400013a6:0x10f7:0x0] mode (null) 00000002:00000001:0.0:1788573204.801638:0:241699:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788573204.801641:0:241699:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788573204.801647:0:241699:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff8fc4a64d9d00 count: 2 00010000:00000001:0.0:1788573204.801651:0:241699:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788573204.801657:0:241699:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.801662:0:241699:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.801665:0:241699:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:0.0:1788573204.801669:0:241699:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x2400013a6:0x10f7:0x0](ffff8fc491dbec08) invalid 00000020:00000001:0.0:1788573204.801680:0:241699:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffff9d48c1483ca0 00000080:00002000:0.0:1788573204.801685:0:241699:0:(llite_internal.h:2029:d_lustre_invalidate()) invalidate dentry d1 (ffff8fc498f1d438) parent ffff8fc498f1d5e8 inode ffff8fc491dbec08 refc 2 00000080:00000001:0.0:1788573204.801689:0:241699:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:0.0:1788573204.801693:0:241699:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788573204.801696:0:241699:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573204.801703:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573204.801705:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573204.801708:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573204.801712:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573204.801715:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573204.801721:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b81e5a00 with handle 0x7d52aadf1286ce27 from hash 00010000:00000001:0.0:1788573204.801732:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573204.801736:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573204.801739:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.801742:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573204.801746:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573204.801749:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573204.801751:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573204.801755:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b86fa200/0x7d52aadf1286ce2e lrc: 2/0,0 mode: PR/PR res: [0x2400013a6:0x10f9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x784a792ceaa5b3aa expref: -99 pid: 265054 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00000001:0.0:1788573204.801795:0:241699:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788573204.801801:0:241699:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788573204.801806:0:241699:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788573204.801842:0:241699:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x2400013a6:0x10f9:0x0] mode (null) 00000002:00000001:0.0:1788573204.801848:0:241699:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788573204.801853:0:241699:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788573204.801860:0:241699:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff8fc4a64d9900 count: 2 00010000:00000001:0.0:1788573204.801867:0:241699:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788573204.801876:0:241699:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.801884:0:241699:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.801891:0:241699:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:0.0:1788573204.801895:0:241699:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x2400013a6:0x10f9:0x0](ffff8fc491dbe3c8) invalid 00000020:00000001:0.0:1788573204.801902:0:241699:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffff9d48c1483ca0 00000080:00002000:0.0:1788573204.801907:0:241699:0:(llite_internal.h:2029:d_lustre_invalidate()) invalidate dentry d3 (ffff8fc491db2bd0) parent ffff8fc498f1d5e8 inode ffff8fc491dbe3c8 refc 4 00000080:00000001:0.0:1788573204.801915:0:241699:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:0.0:1788573204.801917:0:241699:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788573204.801920:0:241699:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573204.801925:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573204.801929:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573204.801932:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573204.801938:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573204.801943:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573204.801947:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b86fa200 with handle 0x7d52aadf1286ce2e from hash 00010000:00000001:0.0:1788573204.801950:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573204.801954:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573204.801963:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.801968:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573204.801974:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573204.801978:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573204.801983:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573204.801988:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b8820600/0x7d52aadf1286ce19 lrc: 2/0,0 mode: PR/PR res: [0x2400013a6:0x10f8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x784a792ceaa5b36b expref: -99 pid: 265053 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00000001:0.0:1788573204.802002:0:241699:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788573204.802007:0:241699:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788573204.802011:0:241699:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788573204.802014:0:241699:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x2400013a6:0x10f8:0x0] mode (null) 00000002:00000001:0.0:1788573204.802020:0:241699:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788573204.802024:0:241699:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788573204.802028:0:241699:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff8fc4a1bc2400 count: 2 00010000:00000001:0.0:1788573204.802033:0:241699:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788573204.802037:0:241699:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.802044:0:241699:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.802049:0:241699:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:0.0:1788573204.802053:0:241699:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x2400013a6:0x10f8:0x0](ffff8fc491dbf448) invalid 00000020:00000001:0.0:1788573204.802059:0:241699:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffff9d48c1483ca0 00000080:00002000:0.0:1788573204.802063:0:241699:0:(llite_internal.h:2029:d_lustre_invalidate()) invalidate dentry d2 (ffff8fc4a0eafa20) parent ffff8fc498f1d5e8 inode ffff8fc491dbf448 refc 3 00000080:00000001:0.0:1788573204.802067:0:241699:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:0.0:1788573204.802070:0:241699:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788573204.802074:0:241699:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573204.802079:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573204.802083:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573204.802087:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573204.802091:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573204.802095:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573204.802101:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b8820600 with handle 0x7d52aadf1286ce19 from hash 00010000:00000001:0.0:1788573204.802107:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573204.802111:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573204.802116:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.802120:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573204.802127:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573204.802136:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573204.802142:0:241699:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1788573204.802146:0:241699:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b81e5000/0x7d52aadf1286ce12 lrc: 2/0,0 mode: PR/PR res: [0x200002b17:0x1057:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x784a792ceaa5b325 expref: -99 pid: 265051 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00000001:0.0:1788573204.802163:0:241699:0:(namei.c:477:ll_md_blocking_ast()) Process entered 00000080:00000001:0.0:1788573204.802168:0:241699:0:(namei.c:240:ll_lock_cancel_bits()) Process entered 00000080:00000001:0.0:1788573204.802172:0:241699:0:(file.c:6096:ll_have_md_lock()) Process entered 00000080:00000040:0.0:1788573204.802176:0:241699:0:(file.c:6102:ll_have_md_lock()) trying to match res [0x200002b17:0x1057:0x0] mode (null) 00000002:00000001:0.0:1788573204.802183:0:241699:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788573204.802188:0:241699:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788573204.802194:0:241699:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff8fc489e3e000 count: 2 00010000:00000001:0.0:1788573204.802199:0:241699:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788573204.802207:0:241699:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1788573204.802213:0:241699:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:0.0:1788573204.802216:0:241699:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1788573204.802222:0:241699:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff8fc489e3e000 count: 2 00010000:00000001:0.0:1788573204.802227:0:241699:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:0.0:1788573204.802234:0:241699:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.802240:0:241699:0:(file.c:6124:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1788573204.802246:0:241699:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:0.0:1788573204.802250:0:241699:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b17:0x1057:0x0](ffff8fc482215348) invalid 00000020:00000001:0.0:1788573204.802257:0:241699:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffff9d48c1483ca0 00000080:00002000:0.0:1788573204.802262:0:241699:0:(llite_internal.h:2029:d_lustre_invalidate()) invalidate dentry d801b.sanity (ffff8fc498f1d5e8) parent ffff8fc4822c06c0 inode ffff8fc482215348 refc 5 00000080:00000001:0.0:1788573204.802272:0:241699:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:0.0:1788573204.802276:0:241699:0:(namei.c:407:ll_lock_cancel_bits()) Process leaving 00000080:00000001:0.0:1788573204.802280:0:241699:0:(namei.c:541:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573204.802285:0:241699:0:(ldlm_lock.c:2501:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1788573204.802289:0:241699:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1788573204.802292:0:241699:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1788573204.802297:0:241699:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1788573204.802301:0:241699:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788573204.802307:0:241699:0:(lustre_handles.c:107:class_handle_unhash_nolock()) removing object ffff8fc4b81e5000 with handle 0x7d52aadf1286ce12 from hash 00010000:00000001:0.0:1788573204.802312:0:241699:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1788573204.802314:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1788573204.802323:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.802327:0:241699:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1788573204.802331:0:241699:0:(ldlm_lock.c:2537:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1788573204.802333:0:241699:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1788573204.802338:0:241699:0:(ldlm_request.c:1890:ldlm_cli_cancel_list_local()) Process leaving (rc=7 : 7 : 7) 00010000:00000001:0.0:1788573204.802342:0:241699:0:(ldlm_request.c:2456:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1788573204.802347:0:241699:0:(ldlm_request.c:1527:ldlm_cli_cancel_req()) Process entered 00000100:00000010:0.0:1788573204.802356:0:241699:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0e00. 00000020:00000040:0.0:1788573204.802363:0:241699:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000100:00000001:0.0:1788573204.802370:0:241699:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573204.802375:0:241699:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573204.802381:0:241699:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573204.802390:0:241699:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8fc485fa8e00. 00000100:00000001:0.0:1788573204.802400:0:241699:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788573204.802406:0:241699:0:(ldlm_request.c:1461:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1788573204.802413:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b81e5400/0x7d52aadf1286ce3c lrc: 1/0,0 mode: --/PR res: [0x2400013a6:0x10fa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b3d4 expref: -99 pid: 265055 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573204.802434:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b81e4200/0x7d52aadf1286ce20 lrc: 1/0,0 mode: --/PR res: [0x2400013a6:0x10f6:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b380 expref: -99 pid: 265051 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573204.802451:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4a635d000/0x7d52aadf1286ce35 lrc: 1/0,0 mode: --/PR res: [0x2400013a6:0x10fb:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b3db expref: -99 pid: 265056 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573204.802464:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b81e5a00/0x7d52aadf1286ce27 lrc: 1/0,0 mode: --/PR res: [0x2400013a6:0x10f7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b3a3 expref: -99 pid: 265052 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573204.802479:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b86fa200/0x7d52aadf1286ce2e lrc: 1/0,0 mode: --/PR res: [0x2400013a6:0x10f9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b3aa expref: -99 pid: 265054 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573204.802494:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b8820600/0x7d52aadf1286ce19 lrc: 1/0,0 mode: --/PR res: [0x2400013a6:0x10f8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b36b expref: -99 pid: 265053 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573204.802526:0:241699:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b81e5000/0x7d52aadf1286ce12 lrc: 1/0,0 mode: --/PR res: [0x200002b17:0x1057:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b325 expref: -99 pid: 265051 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788573204.802538:0:241699:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 7 locks packed 00010000:00000001:0.0:1788573204.802543:0:241699:0:(ldlm_request.c:1500:ldlm_cancel_pack()) Process leaving (rc=7 : 7 : 7) 00000100:00000001:0.0:1788573204.802551:0:241699:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573204.802556:0:241699:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573204.802565:0:241699:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e1d3eedd] to pc [ptlrpcd_00_00+0] req@ffff8fc4b80c0e00 x1875440516543616/t0(0) o103->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:17/18 lens 368/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1788573204.802597:0:241699:0:(ldlm_request.c:1595:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1788573204.802601:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573204.802606:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b81e5400/0x7d52aadf1286ce3c lrc: 0/0,0 mode: --/PR res: [0x2400013a6:0x10fa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b3d4 expref: -99 pid: 265055 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000040:0.0:1788573204.802622:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4a64d9e00 count: 0 00010000:00000010:0.0:1788573204.802630:0:241699:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8fc4b18d7580. 00010000:00000001:0.0:1788573204.802638:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.802644:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573204.802648:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b81e4200/0x7d52aadf1286ce20 lrc: 0/0,0 mode: --/PR res: [0x2400013a6:0x10f6:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b380 expref: -99 pid: 265051 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000040:0.0:1788573204.802666:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc489e3ee00 count: 0 00010000:00000010:0.0:1788573204.802672:0:241699:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8fc482c31200. 00010000:00000001:0.0:1788573204.802680:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.802684:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573204.802689:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4a635d000/0x7d52aadf1286ce35 lrc: 0/0,0 mode: --/PR res: [0x2400013a6:0x10fb:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b3db expref: -99 pid: 265056 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000040:0.0:1788573204.802701:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc489e3e600 count: 0 00010000:00000010:0.0:1788573204.802713:0:241699:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8fc482c31f00. 00010000:00000001:0.0:1788573204.802722:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.802727:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573204.802731:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b81e5a00/0x7d52aadf1286ce27 lrc: 0/0,0 mode: --/PR res: [0x2400013a6:0x10f7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b3a3 expref: -99 pid: 265052 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000040:0.0:1788573204.802768:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4a64d9d00 count: 0 00010000:00000010:0.0:1788573204.802774:0:241699:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8fc4b18d7e80. 00010000:00000001:0.0:1788573204.802788:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.802793:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573204.802798:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b86fa200/0x7d52aadf1286ce2e lrc: 0/0,0 mode: --/PR res: [0x2400013a6:0x10f9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b3aa expref: -99 pid: 265054 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000040:0.0:1788573204.802849:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4a64d9900 count: 0 00010000:00000010:0.0:1788573204.802855:0:241699:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8fc4b18d7c80. 00010000:00000001:0.0:1788573204.802864:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.802868:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573204.802873:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b8820600/0x7d52aadf1286ce19 lrc: 0/0,0 mode: --/PR res: [0x2400013a6:0x10f8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b36b expref: -99 pid: 265053 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000040:0.0:1788573204.802893:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc4a1bc2400 count: 0 00010000:00000010:0.0:1788573204.802899:0:241699:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8fc4aca3b080. 00010000:00000001:0.0:1788573204.802910:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.802912:0:241699:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1788573204.802915:0:241699:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8fc490e72800 lock: ffff8fc4b81e5000/0x7d52aadf1286ce12 lrc: 0/0,0 mode: --/PR res: [0x200002b17:0x1057:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x784a792ceaa5b325 expref: -99 pid: 265051 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000040:0.0:1788573204.802925:0:241699:0:(ldlm_resource.c:1704:ldlm_resource_putref()) putref res: ffff8fc489e3e000 count: 0 00010000:00000010:0.0:1788573204.802931:0:241699:0:(ldlm_resource.c:1565:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8fc482c31a80. 00010000:00000001:0.0:1788573204.802939:0:241699:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1788573204.802942:0:241699:0:(ldlm_request.c:2504:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.0:1788573204.802945:0:241699:0:(ldlm_lockd.c:2919:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff8fc485941840. 00010000:00000001:0.0:1788573204.802955:0:241699:0:(ldlm_lockd.c:2923:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573204.802958:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573204.802962:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573204.802965:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573204.802968:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573204.802973:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573204.802978:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788573204.802982:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573204.802985:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573204.802990:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573205.120947:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573205.120958:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573205.120962:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573205.120972:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573205.120980:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573205.120986:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573205.121141:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573205.121145:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573205.121148:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573205.121181:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573205.121185:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573205.121187:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573205.121190:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573205.121193:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573205.121196:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573205.121199:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573205.121201:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573205.121204:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573206.144909:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573206.144914:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573206.144917:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573206.144923:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573206.144927:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573206.144931:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.144935:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573206.144937:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573206.144944:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573207.168930:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573207.168951:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573207.168954:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573207.168959:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573207.168965:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573207.168969:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573207.168985:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573207.168988:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573207.168991:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573207.169011:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573207.169016:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573207.169018:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573207.169021:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573207.169026:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573207.169028:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573207.169031:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573207.169033:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573207.169035:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573208.192960:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573208.192964:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573208.192967:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573208.192971:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573208.192977:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573208.192983:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573208.192988:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573208.192991:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573208.192994:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573208.448883:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308568301, at 4308568320 00000100:00000010:0.0:1788573208.448900:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7420. 00000020:00000001:0.0:1788573208.896966:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573208.896969:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573208.896972:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573209.216943:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573209.216948:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573209.216951:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573209.216958:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573209.216964:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573209.216968:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573209.216977:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573209.216979:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573209.216984:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573210.240988:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573210.240993:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573210.240996:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573210.241000:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573210.241003:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573210.241006:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573210.241017:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573210.241022:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573210.241026:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573211.264958:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573211.264965:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573211.264971:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573211.264977:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573211.264984:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573211.264990:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.264997:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573211.265001:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573211.265006:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573212.288975:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573212.288981:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573212.288983:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573212.288995:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573212.289008:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573212.289014:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573212.289020:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573212.289026:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573212.289030:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573213.313000:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573213.313006:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573213.313013:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573213.313021:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573213.313026:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573213.313029:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573213.313033:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573213.313038:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573213.313059:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573213.568927:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308573421, at 4308573440 00000100:00000010:0.0:1788573213.568945:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674a20. 00000020:00000001:0.0:1788573214.017054:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573214.017065:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573214.017071:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573214.336977:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573214.336983:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573214.336990:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573214.336997:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573214.337011:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573214.337023:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573214.337046:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573214.337051:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573214.337055:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573215.360972:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573215.360982:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573215.360985:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573215.360992:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573215.361010:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573215.361022:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573215.361027:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573215.361034:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573215.361047:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573216.384953:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573216.384964:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573216.384969:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573216.384979:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573216.384994:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573216.385002:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.385013:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573216.385018:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573216.385026:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573217.408889:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573217.408898:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573217.408901:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573217.408911:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573217.408933:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573217.408937:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573217.408939:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573217.408941:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573217.408943:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573217.408974:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573217.408979:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573217.408983:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573217.408993:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573217.409009:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573217.409014:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573217.409024:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573217.409029:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573217.409032:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573218.432927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573218.432940:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573218.432949:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573218.432962:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573218.432967:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573218.432973:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573218.432978:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573218.432983:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573218.432988:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573218.688947:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308578542, at 4308578560 00000100:00000010:0.0:1788573218.688976:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea75a0. 00000020:00000001:0.0:1788573219.137045:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573219.137050:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573219.137054:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573220.480960:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573220.480967:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573220.480972:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573220.480987:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573220.480995:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573220.480999:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573220.481004:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573220.481010:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573220.481019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573221.504997:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573221.505002:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573221.505005:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573221.505015:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573221.505027:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573221.505037:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.505046:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573221.505050:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573221.505054:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573222.529003:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573222.529010:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573222.529018:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573222.529024:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573222.529034:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573222.529040:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573222.529061:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573222.529065:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573222.529070:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573223.552966:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573223.552984:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573223.552993:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573223.552999:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573223.553005:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573223.553010:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.553021:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573223.553024:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573223.553029:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573223.808925:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308583662, at 4308583680 00000100:00000010:0.0:1788573223.808941:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7540. 00000020:00000001:0.0:1788573224.257066:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573224.257073:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573224.257078:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573224.576958:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573224.576962:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573224.576966:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573224.576971:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573224.576977:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573224.576998:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573224.577005:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573224.577008:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573224.577010:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573226.624969:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573226.624973:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573226.624976:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573226.624980:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573226.624984:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573226.624988:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573226.624995:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573226.624999:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573226.625003:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573227.648898:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573227.648907:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573227.648910:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573227.648917:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573227.648922:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573227.648926:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573227.648930:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573227.648932:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573227.648935:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573227.648965:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573227.648969:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573227.648973:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573227.648976:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573227.648981:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573227.648984:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573227.648988:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573227.648989:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573227.648991:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573228.160920:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788573228.160928:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788573228.160933:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788573228.161104:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573228.161119:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573228.161126:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573228.161145:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.672920:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573228.672928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573228.672931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573228.672940:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573228.672946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573228.672951:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.672956:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573228.672960:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573228.672966:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573228.928922:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308588782, at 4308588800 00000100:00000010:0.0:1788573228.928943:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea79c0. 00000020:00000001:0.0:1788573229.377054:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573229.377060:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573229.377064:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573229.697052:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573229.697059:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573229.697062:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573229.697073:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573229.697083:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573229.697087:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573229.697097:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573229.697101:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573229.697105:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573230.720941:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573230.720953:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573230.720959:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573230.720968:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573230.720984:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573230.720995:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573230.721011:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573230.721021:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573230.721027:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573231.744905:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573231.744912:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573231.744915:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573231.744922:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573231.744943:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573231.744947:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573231.744967:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573231.744971:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573231.744974:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573232.768907:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573232.768915:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573232.768920:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573232.768928:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573232.768935:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573232.768940:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573232.768944:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573232.768948:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573232.768953:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573233.792913:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573233.792920:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573233.792925:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573233.792930:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573233.792937:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573233.792943:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573233.792949:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573233.792954:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573233.792961:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573234.048892:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308593902, at 4308593920 00000100:00000010:0.0:1788573234.048908:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f60. 00000020:00000001:0.0:1788573234.497041:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573234.497046:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573234.497049:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573234.816921:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573234.816931:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573234.816936:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573234.816941:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573234.816946:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573234.816951:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573234.816962:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573234.816967:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573234.816973:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573236.864946:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573236.864954:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573236.864959:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573236.864966:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573236.864973:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573236.864977:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.864997:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573236.865001:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573236.865005:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573237.888952:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573237.888963:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573237.888968:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573237.888979:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573237.888987:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573237.888995:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573237.889003:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573237.889006:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573237.889010:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788573238.401055:0:2388:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788573238.401095:0:2388:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788573238.401103:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573238.401106:0:2388:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788573238.401111:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573238.401115:0:2388:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788573238.401121:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573238.401158:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793700. 00000400:00000200:0.0:1788573238.401165:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573238.401170:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573238.401177:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc660 00000400:00000010:0.0:1788573238.401180:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc660. 00000100:00000001:0.0:1788573238.401185:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573238.401194:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac821c00 x1875440516547072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573260 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.401220:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.401240:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516547072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573260 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.401250:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.401253:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573238.401265:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573238.401269:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793000. 00000400:00000200:0.0:1788573238.401272:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573238.401278:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573238.401280:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fce58 00000400:00000010:0.0:1788573238.401284:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fce58. 00000100:00000001:0.0:1788573238.401287:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573238.401294:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac820e00 x1875440516547200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573260 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.401302:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.401306:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516547200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573260 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.401325:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.401329:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573238.401338:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573238.401344:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793a00. 00000400:00000200:0.0:1788573238.401349:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573238.401358:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573238.401364:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc440 00000400:00000010:0.0:1788573238.401369:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc440. 00000100:00000001:0.0:1788573238.401374:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573238.401383:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac820a80 x1875440516547328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573260 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.401400:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.401409:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516547328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573260 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.401423:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.401435:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573238.401447:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573238.401452:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793800. 00000400:00000200:0.0:1788573238.401457:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573238.401476:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573238.401481:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcdd0 00000400:00000010:0.0:1788573238.401484:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcdd0. 00000100:00000001:0.0:1788573238.401491:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573238.401501:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac821880 x1875440516547456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573260 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.401519:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.401527:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516547456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573260 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.401541:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.401546:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573238.401557:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573238.401562:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793300. 00000400:00000200:0.0:1788573238.401568:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573238.401578:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573238.401585:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcaa0 00000400:00000010:0.0:1788573238.401588:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcaa0. 00000100:00000001:0.0:1788573238.401593:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573238.401601:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac823800 x1875440516547584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573260 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.401616:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573238.401625:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516547584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573260 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573238.401641:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.401645:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573238.912917:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573238.912925:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573238.912927:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573238.912952:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573238.912958:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573238.912964:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.912975:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573238.912984:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.913002:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573238.913050:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573238.913057:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573238.913061:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573238.913064:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573238.913068:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573238.913069:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.913072:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573238.913073:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573238.913075:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573239.168922:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308599021, at 4308599040 00000100:00000010:0.0:1788573239.168942:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea77e0. 00000020:00000001:0.0:1788573239.617010:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573239.617015:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573239.617018:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573239.936950:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573239.936955:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573239.936961:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573239.936968:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573239.936975:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573239.936979:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573239.936988:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573239.936991:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573239.936997:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573240.448955:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573240.448967:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573240.448970:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573240.448974:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573240.448978:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573240.448986:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13927 00000400:00000200:0.0:1788573240.448995:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573240.449016:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573240.449019:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573240.449023:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449026:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573240.449029:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449031:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573240.449035:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573240.449038:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573240.449041:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573240.449044:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573240.449047:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573240.449050:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573240.449053:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449055:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573240.449057:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449059:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573240.449063:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573240.449066:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449069:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573240.449072:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573240.449075:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573240.449078:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449081:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573240.449083:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573240.449103:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573240.449107:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573240.449111:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573240.449113:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449116:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573240.449118:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573240.449123:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573240.449130:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573240.449143:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573240.449153:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516547712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573240.449165:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573240.449169:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573240.449188:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516547712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573240.449215:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573240.449218:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573240.449222:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573240.449224:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573240.449226:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573240.449228:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573240.449231:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13927 00000400:00000200:0.0:1788573240.449236:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573240.449239:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573240.449242:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573240.449245:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449248:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573240.449250:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449252:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573240.449254:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573240.449256:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573240.449258:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573240.449261:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573240.449263:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573240.449265:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573240.449268:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449271:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573240.449278:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449280:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573240.449283:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573240.449286:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449289:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573240.449291:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573240.449298:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573240.449300:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449303:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573240.449308:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573240.449316:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449319:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573240.449323:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573240.449326:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573240.449328:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573240.449330:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449332:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573240.449333:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573240.449335:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573240.449340:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573240.449344:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573240.449349:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516547840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573240.449358:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573240.449361:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573240.449364:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516547840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573240.449374:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573240.449377:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573240.449380:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573240.449382:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573240.449392:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573240.449395:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573240.449398:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13927 00000400:00000200:0.0:1788573240.449402:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573240.449406:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573240.449408:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573240.449411:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449414:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573240.449416:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449419:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573240.449422:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573240.449423:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573240.449424:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573240.449427:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573240.449429:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573240.449431:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573240.449434:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449436:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573240.449438:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449441:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573240.449443:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573240.449446:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449449:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573240.449450:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573240.449453:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573240.449455:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449457:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573240.449460:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573240.449463:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449465:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573240.449472:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573240.449474:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573240.449477:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573240.449479:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449481:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573240.449482:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573240.449484:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573240.449488:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573240.449492:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573240.449497:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516547968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573240.449505:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573240.449507:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573240.449511:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516547968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573240.449519:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573240.449522:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573240.449525:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573240.449527:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573240.449529:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573240.449532:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573240.449534:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13927 00000400:00000200:0.0:1788573240.449538:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573240.449541:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573240.449543:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573240.449547:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449549:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573240.449552:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449554:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573240.449556:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573240.449561:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573240.449563:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573240.449565:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573240.449567:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573240.449570:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573240.449573:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449575:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573240.449577:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449579:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573240.449582:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573240.449584:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449587:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573240.449589:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573240.449591:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573240.449594:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449596:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573240.449599:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573240.449601:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449603:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573240.449606:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573240.449609:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573240.449611:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573240.449613:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449615:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573240.449616:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573240.449618:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573240.449621:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573240.449624:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573240.449629:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516548096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573240.449637:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573240.449639:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573240.449647:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516548096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573240.449656:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573240.449659:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573240.449662:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573240.449664:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573240.449665:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573240.449668:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573240.449671:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13927 00000400:00000200:0.0:1788573240.449675:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573240.449678:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573240.449681:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573240.449683:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449686:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573240.449688:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449690:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573240.449693:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573240.449694:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573240.449696:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573240.449698:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573240.449700:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573240.449702:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573240.449705:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449707:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573240.449709:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449712:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573240.449714:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573240.449717:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573240.449719:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573240.449725:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573240.449727:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573240.449730:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449732:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573240.449734:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573240.449736:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449738:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573240.449742:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573240.449744:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573240.449747:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573240.449748:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573240.449750:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573240.449751:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573240.449753:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573240.449756:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573240.449760:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573240.449765:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516548224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573240.449772:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573240.449774:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573240.449779:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516548224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573240.449787:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573240.449792:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13938) 00000100:00000001:0.0:1788573240.960907:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573240.960913:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573240.960917:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573240.960921:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573240.960925:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573240.960930:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573240.960938:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573240.960942:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573240.960946:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573241.984957:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573241.984983:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573241.984994:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573241.985011:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573241.985026:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573241.985035:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573241.985077:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573241.985080:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573241.985085:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573243.008983:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573243.008988:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573243.008991:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573243.008995:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573243.009001:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573243.009006:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573243.009009:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573243.009010:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573243.009013:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573244.032918:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573244.032927:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573244.032931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573244.032937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573244.032941:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573244.032946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573244.032958:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573244.032962:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573244.032966:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573244.288898:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308604142, at 4308604160 00000100:00000010:0.0:1788573244.288920:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7540. 00000020:00000001:0.0:1788573244.737028:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573244.737032:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573244.737034:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573245.056918:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573245.056923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573245.056928:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573245.056933:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573245.056938:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573245.056942:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573245.056956:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573245.056959:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573245.056963:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573246.080976:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573246.081000:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573246.081004:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573246.081021:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573246.081036:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573246.081045:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573246.081053:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573246.081055:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573246.081058:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573247.104892:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573247.104901:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573247.104905:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573247.104919:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573247.104923:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573247.104927:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573247.105135:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573247.105138:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573247.105142:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573247.105175:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573247.105179:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573247.105180:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573247.105182:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573247.105185:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573247.105188:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573247.105189:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573247.105190:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573247.105192:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573248.128932:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573248.128938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573248.128960:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573248.128968:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573248.128976:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573248.128982:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573248.129004:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573248.129008:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573248.129023:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573249.152935:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573249.152941:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573249.152945:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573249.152952:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573249.152967:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573249.152978:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573249.152995:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573249.153005:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573249.153017:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573249.408945:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308609262, at 4308609280 00000100:00000010:0.0:1788573249.408958:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916745a0. 00000020:00000001:0.0:1788573249.856994:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573249.856998:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573249.857001:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573250.176957:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573250.176964:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573250.176968:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573250.176974:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573250.176982:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573250.176987:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573250.177010:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573250.177015:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573250.177020:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573250.177042:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573250.177045:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573250.177047:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573250.177050:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573250.177054:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573250.177057:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573250.177060:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573250.177062:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573250.177064:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573251.201008:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573251.201016:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573251.201026:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573251.201037:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573251.201070:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573251.201076:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573251.201093:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573251.201098:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573251.201104:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573252.224935:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573252.224948:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573252.224951:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573252.224956:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573252.224963:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573252.224970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573252.224990:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573252.224993:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573252.224996:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573252.225084:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573252.225089:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573252.225091:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573252.225095:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573252.225101:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573252.225105:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573252.225109:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573252.225111:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573252.225114:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573253.249014:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573253.249021:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573253.249027:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573253.249037:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573253.249047:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573253.249054:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573253.249060:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573253.249066:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573253.249074:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573254.272933:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573254.272942:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573254.272944:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573254.272951:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573254.272958:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573254.272965:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573254.273000:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573254.273004:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573254.273008:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573254.528939:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308614381, at 4308614400 00000100:00000010:0.0:1788573254.528963:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916744e0. 00000020:00000001:0.0:1788573254.977092:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573254.977098:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573254.977102:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573255.296937:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573255.296945:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573255.296947:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573255.296952:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573255.296956:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573255.296960:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573255.296967:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573255.296972:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573255.296978:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573255.297012:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573255.297018:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573255.297022:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573255.297029:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573255.297034:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573255.297040:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573255.297052:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573255.297055:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573255.297060:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573256.321002:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573256.321013:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573256.321020:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573256.321037:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573256.321046:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573256.321051:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573256.321066:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573256.321069:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573256.321071:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573257.344941:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573257.344950:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573257.344985:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573257.344996:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573257.345006:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573257.345015:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573257.345049:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573257.345058:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573257.345067:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573258.369014:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573258.369021:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573258.369023:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573258.369027:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573258.369031:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573258.369035:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573258.369041:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573258.369046:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573258.369050:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573259.392976:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573259.392988:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573259.392994:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573259.393014:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573259.393027:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573259.393041:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573259.393060:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573259.393066:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573259.393071:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573259.648907:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308619502, at 4308619520 00000100:00000010:0.0:1788573259.648922:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea75a0. 00000020:00000001:0.0:1788573260.097050:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573260.097055:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573260.097058:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573260.416953:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573260.416971:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573260.416983:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573260.416999:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573260.417005:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573260.417013:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.417022:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573260.417049:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.417053:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573260.417081:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573260.417087:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573260.417088:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573260.417091:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573260.417095:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573260.417098:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.417102:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573260.417104:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573260.417107:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573261.440963:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573261.440970:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573261.440974:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573261.440982:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573261.440990:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573261.440996:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573261.441012:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573261.441018:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573261.441022:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573262.464894:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573262.464902:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573262.464906:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573262.464915:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573262.464923:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573262.464928:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573262.464938:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573262.464942:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573262.464946:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573263.488922:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573263.488935:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573263.488943:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573263.488955:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573263.488960:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573263.488968:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573263.488994:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573263.488999:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573263.489006:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573264.512935:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573264.512939:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573264.512943:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573264.512946:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573264.512953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573264.512958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573264.512961:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573264.512963:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573264.512968:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573264.768899:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308624621, at 4308624640 00000100:00000010:0.0:1788573264.768916:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916741e0. 00000020:00000001:0.0:1788573265.217020:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573265.217024:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573265.217028:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573265.536935:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573265.536951:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573265.536962:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573265.536983:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573265.537002:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573265.537021:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573265.537040:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573265.537044:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573265.537047:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573265.537100:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573265.537106:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573265.537108:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573265.537113:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573265.537117:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573265.537123:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573265.537127:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573265.537130:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573265.537134:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573266.048936:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573266.048946:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573266.048950:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573266.048954:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573266.048983:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573266.048987:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13953 00000400:00000200:0.0:1788573266.048997:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573266.049004:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573266.049007:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573266.049009:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049012:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573266.049014:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049017:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573266.049020:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573266.049022:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573266.049024:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573266.049027:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573266.049029:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573266.049031:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573266.049033:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049034:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573266.049036:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049038:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573266.049040:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573266.049043:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049045:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573266.049047:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573266.049050:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573266.049052:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.049054:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573266.049056:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573266.049063:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573266.049066:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573266.049070:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573266.049071:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.049078:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573266.049081:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573266.049087:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573266.049096:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573266.049103:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573266.049114:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516550912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573266.049126:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573266.049132:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573266.049137:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516550912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573266.049164:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573266.049168:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573266.049171:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573266.049172:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573266.049174:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573266.049184:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573266.049187:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13953 00000400:00000200:0.0:1788573266.049191:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573266.049193:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573266.049196:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573266.049200:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049203:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573266.049205:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049210:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573266.049214:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573266.049215:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573266.049217:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573266.049219:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573266.049221:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573266.049224:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573266.049243:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049245:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573266.049247:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049248:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573266.049250:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573266.049254:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049260:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573266.049261:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573266.049268:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573266.049273:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.049275:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573266.049281:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573266.049284:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.049288:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573266.049295:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573266.049300:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573266.049302:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573266.049302:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.049304:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573266.049308:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573266.049310:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573266.049318:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573266.049324:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573266.049329:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516551040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573266.049341:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573266.049343:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573266.049350:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516551040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573266.049358:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573266.049362:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573266.049372:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573266.049375:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573266.049376:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573266.049380:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573266.049385:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13953 00000400:00000200:0.0:1788573266.049390:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573266.049394:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573266.049399:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573266.049402:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049405:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573266.049409:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049415:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573266.049419:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573266.049421:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573266.049424:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573266.049426:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573266.049429:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573266.049433:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573266.049439:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049443:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573266.049444:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049446:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573266.049450:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573266.049453:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049455:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573266.049459:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573266.049463:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573266.049465:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.049468:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573266.049471:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573266.049478:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.049482:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573266.049486:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573266.049490:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573266.049492:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573266.049494:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.049496:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573266.049499:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573266.049503:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573266.049507:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573266.049513:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573266.049519:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516551168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573266.049534:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573266.049537:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573266.049544:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516551168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573266.049558:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573266.049563:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573266.049566:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573266.049567:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573266.049569:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573266.049573:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573266.049579:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13953 00000400:00000200:0.0:1788573266.049598:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573266.049606:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573266.049609:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573266.049611:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049613:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573266.049630:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049646:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573266.049652:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573266.049655:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573266.049660:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573266.049665:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573266.049672:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573266.049678:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573266.049687:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049693:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573266.049699:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049704:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573266.049710:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573266.049717:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573266.049721:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573266.049726:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573266.049732:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573266.049737:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.049742:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573266.049757:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573266.049762:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.049771:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573266.049780:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573266.049785:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573266.049791:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573266.049794:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.049797:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573266.049800:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573266.049804:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573266.049854:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573266.049862:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573266.049872:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516551296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573266.049906:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573266.049911:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573266.049922:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516551296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573266.049948:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573266.049951:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573266.049955:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573266.049957:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573266.049960:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573266.049968:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573266.049974:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13953 00000400:00000200:0.0:1788573266.049982:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573266.049989:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573266.049995:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573266.050003:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.050007:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573266.050015:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.050020:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573266.050027:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573266.050030:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573266.050035:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573266.050041:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573266.050048:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573266.050052:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573266.050060:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.050064:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573266.050070:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573266.050075:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573266.050087:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573266.050097:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573266.050102:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573266.050107:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573266.050111:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573266.050119:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.050122:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573266.050125:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573266.050128:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.050130:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573266.050134:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573266.050137:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573266.050140:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573266.050142:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.050144:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573266.050146:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573266.050149:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573266.050153:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573266.050158:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573266.050163:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516551424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573266.050171:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573266.050173:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573266.050178:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516551424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573266.050186:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573266.050192:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13963) 00000100:00000001:0.0:1788573266.560903:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573266.560908:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573266.560911:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573266.560917:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573266.560923:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573266.560944:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573266.560949:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573266.560953:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573266.560960:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573266.561018:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573266.561021:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573266.561023:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573266.561025:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573266.561028:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573266.561030:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573266.561031:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573266.561033:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573266.561035:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573267.584919:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573267.584929:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573267.584932:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573267.584939:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573267.584945:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573267.584950:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573267.584954:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573267.584959:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573267.584963:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573267.584979:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573267.584983:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573267.584986:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573267.584989:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573267.584994:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573267.584996:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573267.585000:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573267.585001:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573267.585003:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573268.608961:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573268.608970:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573268.608978:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573268.608984:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573268.608997:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573268.609009:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573268.609017:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573268.609033:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573268.609042:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573269.888942:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308629742, at 4308629760 00000100:00000010:0.0:1788573269.888962:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7b40. 00000020:00000001:0.0:1788573270.337110:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573270.337116:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573270.337119:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573270.657010:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573270.657017:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573270.657022:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573270.657027:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573270.657035:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573270.657044:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573270.657066:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573270.657073:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573270.657088:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573271.169029:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573271.169041:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573271.169044:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573271.169048:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573271.169053:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573271.169061:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13958 00000400:00000200:0.0:1788573271.169071:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573271.169079:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573271.169083:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573271.169088:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169092:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573271.169095:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169098:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573271.169103:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573271.169106:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573271.169108:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573271.169111:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573271.169130:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573271.169132:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573271.169135:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169137:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573271.169138:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169140:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573271.169142:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573271.169144:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169146:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573271.169148:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573271.169151:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573271.169153:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.169154:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573271.169156:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573271.169162:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573271.169164:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573271.169168:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573271.169170:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.169172:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573271.169174:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573271.169179:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573271.169186:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573271.169193:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573271.169203:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516551552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573271.169214:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573271.169219:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573271.169224:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516551552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573271.169248:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573271.169251:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573271.169262:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573271.169264:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573271.169266:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573271.169270:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573271.169273:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13958 00000400:00000200:0.0:1788573271.169278:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573271.169283:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573271.169289:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573271.169295:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169299:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573271.169304:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169307:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573271.169312:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573271.169313:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573271.169315:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573271.169318:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573271.169324:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573271.169329:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573271.169336:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169340:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573271.169345:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169349:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573271.169355:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573271.169361:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169366:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573271.169371:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573271.169379:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573271.169383:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.169387:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573271.169391:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573271.169405:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.169407:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573271.169412:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573271.169417:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573271.169423:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573271.169425:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.169426:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573271.169427:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573271.169429:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573271.169436:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573271.169440:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573271.169449:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516551680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573271.169460:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573271.169463:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573271.169471:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516551680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573271.169485:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573271.169491:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573271.169496:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573271.169497:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573271.169499:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573271.169501:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573271.169504:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13958 00000400:00000200:0.0:1788573271.169510:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573271.169514:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573271.169518:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573271.169520:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169522:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573271.169527:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169538:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573271.169542:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573271.169543:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573271.169545:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573271.169546:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573271.169550:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573271.169554:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573271.169574:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169577:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573271.169582:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169584:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573271.169587:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573271.169593:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169596:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573271.169598:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573271.169605:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573271.169610:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.169615:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573271.169619:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573271.169624:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.169629:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573271.169635:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573271.169641:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573271.169646:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573271.169650:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.169654:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573271.169657:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573271.169660:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573271.169666:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573271.169674:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573271.169681:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516551808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573271.169705:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573271.169708:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573271.169711:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516551808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573271.169723:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573271.169727:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573271.169732:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573271.169734:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573271.169737:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573271.169740:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573271.169744:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13958 00000400:00000200:0.0:1788573271.169748:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573271.169754:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573271.169758:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573271.169763:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169766:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573271.169768:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169773:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573271.169776:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573271.169779:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573271.169781:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573271.169783:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573271.169786:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573271.169788:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573271.169792:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169794:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573271.169797:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169802:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573271.169839:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573271.169846:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573271.169851:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573271.169856:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573271.169860:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573271.169864:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.169871:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573271.169876:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573271.169880:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.169885:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573271.169891:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573271.169895:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573271.169899:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573271.169903:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.169908:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573271.169911:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573271.169915:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573271.169923:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573271.169929:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573271.169939:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516551936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573271.169960:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573271.169968:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573271.169980:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516551936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573271.169997:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573271.170002:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573271.170010:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573271.170013:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573271.170017:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573271.170022:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573271.170033:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13958 00000400:00000200:0.0:1788573271.170042:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573271.170048:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573271.170052:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573271.170056:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.170060:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573271.170066:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.170072:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573271.170077:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573271.170080:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573271.170085:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573271.170088:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573271.170093:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573271.170099:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573271.170107:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.170112:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573271.170116:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573271.170118:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573271.170120:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573271.170122:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573271.170126:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573271.170130:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573271.170133:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573271.170138:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.170142:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573271.170144:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573271.170149:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.170152:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573271.170156:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573271.170161:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573271.170165:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573271.170171:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573271.170174:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573271.170176:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573271.170181:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573271.170185:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573271.170190:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573271.170198:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516552064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573271.170209:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573271.170212:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573271.170220:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516552064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573271.170233:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573271.170239:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13968) 00000100:00000001:0.0:1788573272.705032:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573272.705044:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573272.705049:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573272.705057:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573272.705063:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573272.705069:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573272.705079:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573272.705084:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573272.705088:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573273.728926:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573273.728931:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573273.728935:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573273.728942:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573273.728947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573273.728951:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573273.728966:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573273.728969:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573273.728972:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573274.752897:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573274.752903:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573274.752906:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573274.752912:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573274.752929:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573274.752933:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573274.752946:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573274.752949:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573274.752952:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573275.008911:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308634862, at 4308634880 00000100:00000010:0.0:1788573275.008929:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7600. 00000020:00000001:0.0:1788573275.456997:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573275.457005:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573275.457010:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573275.776937:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573275.776945:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573275.776951:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573275.776959:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573275.776965:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573275.776973:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573275.776987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573275.776991:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573275.776994:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573275.777011:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573275.777016:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573275.777018:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573275.777021:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573275.777025:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573275.777028:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573275.777030:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573275.777032:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573275.777034:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573276.800989:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573276.800996:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573276.800999:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573276.801006:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573276.801013:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573276.801021:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573276.801026:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573276.801028:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573276.801031:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573277.824905:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573277.824911:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573277.824914:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573277.824921:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573277.824927:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573277.824931:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573277.824948:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573277.824952:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573277.824955:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573278.848915:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573278.848924:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573278.848928:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573278.848936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573278.848946:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573278.848952:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573278.848976:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573278.848979:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573278.848986:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573278.849017:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573278.849020:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573278.849022:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573278.849025:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573278.849027:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573278.849029:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573278.849031:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573278.849033:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573278.849035:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573279.872948:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573279.872955:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573279.872958:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573279.872963:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573279.872968:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573279.872973:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573279.872990:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573279.872993:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573279.872996:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573280.128913:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308639981, at 4308640000 00000100:00000010:0.0:1788573280.128932:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674f00. 00000020:00000001:0.0:1788573280.577013:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573280.577017:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573280.577020:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573280.896968:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573280.896976:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573280.896982:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573280.896992:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573280.897007:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573280.897015:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573280.897027:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573280.897030:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573280.897035:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573281.920937:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573281.920942:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573281.920947:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573281.920957:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573281.920972:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573281.920978:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573281.921016:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573281.921019:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573281.921022:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573282.944950:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573282.944962:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573282.944966:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573282.944973:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573282.944982:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573282.944990:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573282.945003:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573282.945010:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573282.945022:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573283.968989:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573283.968998:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573283.969003:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573283.969019:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573283.969032:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573283.969037:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573283.969043:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573283.969049:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573283.969066:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573284.992891:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573284.992898:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573284.992900:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573284.992905:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573284.992910:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573284.992914:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573284.992919:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573284.992923:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573284.992928:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573284.992974:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573284.992979:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573284.992986:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573284.993000:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573284.993014:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573284.993016:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573284.993020:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573284.993024:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573284.993034:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573285.249009:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308645101, at 4308645120 00000100:00000010:0.0:1788573285.249022:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674ea0. 00000020:00000001:0.0:1788573285.697071:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573285.697078:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573285.697082:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573286.528906:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573286.528918:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573286.528921:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573286.528924:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573286.528930:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573286.528937:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13973 00000400:00000200:0.0:1788573286.528945:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573286.528951:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573286.528954:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573286.528957:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.528978:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573286.528980:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.528982:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573286.528986:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573286.528988:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573286.528991:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573286.528994:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573286.528996:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573286.528998:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573286.529001:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529002:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573286.529004:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529006:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573286.529009:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573286.529013:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529015:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573286.529018:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573286.529020:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573286.529023:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.529025:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573286.529027:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573286.529032:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573286.529036:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573286.529039:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573286.529041:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.529043:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573286.529046:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573286.529050:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573286.529057:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573286.529063:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573286.529071:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516553472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573286.529091:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573286.529095:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573286.529100:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516553472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573286.529121:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573286.529124:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573286.529127:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573286.529129:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573286.529130:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573286.529133:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573286.529135:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13973 00000400:00000200:0.0:1788573286.529139:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573286.529142:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573286.529144:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573286.529147:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529150:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573286.529152:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529154:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573286.529156:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573286.529158:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573286.529160:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573286.529162:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573286.529164:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573286.529167:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573286.529171:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529173:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573286.529176:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529179:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573286.529188:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573286.529196:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529201:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573286.529206:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573286.529221:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573286.529230:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.529238:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573286.529245:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573286.529253:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.529259:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573286.529266:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573286.529272:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573286.529277:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573286.529280:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.529284:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573286.529289:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573286.529294:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573286.529300:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573286.529307:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573286.529320:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516553600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573286.529338:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573286.529345:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573286.529358:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516553600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573286.529391:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573286.529397:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573286.529401:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573286.529403:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573286.529404:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573286.529407:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573286.529419:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13973 00000400:00000200:0.0:1788573286.529427:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573286.529437:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573286.529447:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573286.529456:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529462:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573286.529469:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529477:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573286.529482:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573286.529485:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573286.529487:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573286.529489:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573286.529495:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573286.529499:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573286.529509:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529514:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573286.529517:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529523:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573286.529529:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573286.529541:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529547:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573286.529554:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573286.529562:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573286.529571:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.529578:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573286.529586:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573286.529597:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.529600:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573286.529604:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573286.529610:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573286.529618:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573286.529630:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.529635:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573286.529639:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573286.529643:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573286.529650:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573286.529656:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573286.529670:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516553728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573286.529702:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573286.529706:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573286.529715:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516553728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573286.529744:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573286.529753:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573286.529759:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573286.529765:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573286.529772:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573286.529781:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573286.529789:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13973 00000400:00000200:0.0:1788573286.529802:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573286.529852:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573286.529855:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573286.529866:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529874:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573286.529883:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529890:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573286.529898:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573286.529900:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573286.529903:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573286.529906:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573286.529914:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573286.529924:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573286.529931:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529936:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573286.529939:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529946:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573286.529949:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573286.529956:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573286.529961:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573286.529964:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573286.529972:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573286.529980:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.529986:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573286.529990:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573286.529996:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.530001:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573286.530007:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573286.530013:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573286.530024:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573286.530030:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.530035:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573286.530038:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573286.530043:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573286.530051:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573286.530056:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573286.530072:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516553856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573286.530120:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573286.530126:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573286.530139:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516553856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573286.530166:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573286.530175:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573286.530182:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573286.530185:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573286.530189:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573286.530203:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573286.530214:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13973 00000400:00000200:0.0:1788573286.530227:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573286.530235:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573286.530246:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573286.530256:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.530266:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573286.530275:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.530281:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573286.530288:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573286.530292:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573286.530297:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573286.530302:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573286.530311:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573286.530317:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573286.530329:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.530336:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573286.530348:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573286.530357:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573286.530364:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573286.530386:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573286.530393:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573286.530398:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573286.530404:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573286.530413:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.530422:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573286.530434:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573286.530441:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.530446:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573286.530452:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573286.530457:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573286.530464:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573286.530468:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573286.530471:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573286.530472:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573286.530474:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573286.530483:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573286.530491:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573286.530503:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516553984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573286.530516:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573286.530521:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573286.530529:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516553984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573286.530561:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573286.530569:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13984) 00000100:00000001:0.0:1788573287.040945:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573287.040951:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573287.040953:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573287.040959:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573287.040964:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573287.040968:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573287.040976:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573287.040979:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573287.040983:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573288.064910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573288.064917:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573288.064922:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573288.064929:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573288.064935:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573288.064954:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573288.064960:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573288.064964:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573288.064969:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573289.088889:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573289.088897:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573289.088900:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573289.088908:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573289.088916:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573289.088922:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573289.088931:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573289.088938:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573289.088943:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573290.112955:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573290.112959:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573290.112962:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573290.112968:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573290.112982:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573290.112984:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573290.112993:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573290.112997:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573290.113001:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573290.368893:0:265575:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308650222, at 4308650240 00000100:00000010:0.0:1788573290.368916:0:265575:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674840. 00000020:00000001:0.0:1788573290.817013:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573290.817017:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573290.817019:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573291.136974:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573291.136978:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573291.136979:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573291.136983:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573291.136988:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573291.136994:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573291.137007:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573291.137010:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573291.137014:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573292.160963:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573292.160988:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573292.160993:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573292.161002:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573292.161010:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573292.161016:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573292.161023:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573292.161027:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573292.161031:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573293.184953:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573293.184959:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573293.184964:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573293.184970:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573293.184977:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573293.184981:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573293.184996:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573293.184999:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573293.185003:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573294.209054:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573294.209059:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573294.209061:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573294.209064:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573294.209068:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573294.209070:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573294.209074:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573294.209076:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573294.209078:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573295.232945:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573295.232956:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573295.232961:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573295.232970:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573295.232979:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573295.232985:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573295.233146:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573295.233150:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573295.233158:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573295.488883:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308655341, at 4308655360 00000100:00000010:0.0:1788573295.488903:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c00. 00000020:00000001:0.0:1788573295.936996:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573295.937018:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573295.937022:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573296.256950:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573296.256956:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573296.256961:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573296.256966:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573296.256972:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573296.256984:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573296.256987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573296.256990:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573296.256994:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573297.280925:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573297.280933:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573297.280937:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573297.280947:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573297.280957:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573297.280964:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573297.280971:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573297.280975:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573297.280979:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573298.304943:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573298.304949:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573298.304954:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573298.304960:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573298.304967:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573298.304973:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573298.305022:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573298.305027:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573298.305033:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573299.328923:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573299.328933:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573299.328936:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573299.328945:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573299.328953:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573299.328960:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573299.328967:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573299.328972:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573299.328979:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573300.352971:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573300.352985:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573300.352997:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573300.353005:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573300.353019:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573300.353029:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573300.353078:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573300.353082:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573300.353088:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573300.608878:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308660461, at 4308660480 00000100:00000010:0.0:1788573300.608898:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7d80. 00000020:00000001:0.0:1788573301.057007:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573301.057011:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573301.057014:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573301.377175:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573301.377183:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573301.377186:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573301.377217:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573301.377225:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573301.377235:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573301.377245:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573301.377248:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573301.377255:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573301.888982:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573301.888995:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573301.888999:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573301.889003:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573301.889008:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573301.889016:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 13989 00000400:00000200:0.0:1788573301.889025:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573301.889032:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573301.889035:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573301.889039:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889042:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573301.889064:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889067:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573301.889071:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573301.889073:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573301.889076:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573301.889080:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573301.889082:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573301.889085:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573301.889088:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889090:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573301.889093:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889095:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573301.889098:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573301.889102:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889105:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573301.889108:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573301.889112:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573301.889115:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.889118:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573301.889120:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573301.889127:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573301.889131:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573301.889135:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573301.889137:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.889140:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573301.889143:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573301.889148:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573301.889155:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573301.889163:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573301.889171:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516555392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573301.889183:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573301.889209:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573301.889216:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516555392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573301.889248:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573301.889251:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573301.889255:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573301.889257:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573301.889259:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573301.889262:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573301.889265:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13989 00000400:00000200:0.0:1788573301.889270:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573301.889274:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573301.889277:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573301.889281:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889284:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573301.889287:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889290:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573301.889295:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573301.889305:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573301.889310:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573301.889315:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573301.889321:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573301.889328:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573301.889338:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889344:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573301.889350:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889357:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573301.889363:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573301.889370:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889385:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573301.889389:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573301.889399:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573301.889404:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.889409:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573301.889416:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573301.889424:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.889429:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573301.889436:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573301.889441:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573301.889444:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573301.889447:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.889451:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573301.889455:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573301.889460:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573301.889467:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573301.889476:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573301.889488:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516555520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573301.889509:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573301.889514:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573301.889524:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516555520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573301.889543:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573301.889550:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573301.889558:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573301.889563:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573301.889565:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573301.889568:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573301.889574:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13989 00000400:00000200:0.0:1788573301.889582:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573301.889607:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573301.889612:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573301.889623:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889629:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573301.889634:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889639:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573301.889643:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573301.889646:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573301.889650:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573301.889655:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573301.889659:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573301.889665:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573301.889672:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889676:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573301.889679:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889684:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573301.889690:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573301.889696:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889701:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573301.889703:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573301.889707:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573301.889712:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.889718:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573301.889722:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573301.889730:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.889736:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573301.889745:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573301.889752:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573301.889759:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573301.889762:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.889768:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573301.889778:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573301.889783:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573301.889792:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573301.889800:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573301.889847:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516555648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573301.889867:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573301.889874:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573301.889883:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516555648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573301.889906:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573301.889910:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573301.889917:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573301.889921:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573301.889924:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573301.889930:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573301.889936:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13989 00000400:00000200:0.0:1788573301.889968:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573301.889974:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573301.889979:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573301.889984:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.889988:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573301.889994:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.890000:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573301.890011:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573301.890015:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573301.890018:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573301.890023:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573301.890029:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573301.890035:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573301.890052:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.890059:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573301.890065:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.890071:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573301.890077:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573301.890083:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573301.890091:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573301.890096:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573301.890102:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573301.890108:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.890112:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573301.890117:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573301.890123:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.890127:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573301.890133:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573301.890139:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573301.890145:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573301.890149:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.890155:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573301.890159:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573301.890163:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573301.890168:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573301.890177:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573301.890188:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516555776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573301.890219:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573301.890224:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573301.890233:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516555776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573301.890254:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573301.890259:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573301.890277:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573301.890282:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573301.890286:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573301.890292:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573301.890298:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 13989 00000400:00000200:0.0:1788573301.890307:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573301.890315:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573301.890319:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573301.890326:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.890332:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573301.890342:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.890350:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573301.890357:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573301.890361:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573301.890365:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573301.890371:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573301.890376:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573301.890383:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573301.890392:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.890396:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573301.890402:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573301.890407:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573301.890413:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573301.890422:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573301.890426:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573301.890429:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573301.890435:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573301.890438:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.890439:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573301.890444:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573301.890458:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.890461:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573301.890468:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573301.890475:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573301.890480:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573301.890483:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573301.890487:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573301.890490:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573301.890494:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573301.890500:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573301.890508:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573301.890518:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516555904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573301.890534:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573301.890538:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573301.890545:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516555904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573301.890564:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573301.890573:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13999) 00000100:00000001:0.0:1788573302.400975:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573302.400981:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573302.400985:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573302.400992:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573302.400998:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573302.401000:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573302.401171:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573302.401175:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573302.401178:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573303.424911:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573303.424919:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573303.424922:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573303.424928:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573303.424935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573303.424940:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573303.424969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573303.424972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573303.424976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.448912:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573304.448921:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573304.448925:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573304.448932:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573304.448939:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573304.448943:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573304.448949:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573304.448953:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573304.448958:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573304.448988:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573304.448994:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573304.448999:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573304.449006:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573304.449011:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573304.449017:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573304.449025:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573304.449028:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573304.449031:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573304.960913:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788573304.960924:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788573304.960930:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:0.0:1788573305.472919:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573305.472926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573305.472929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573305.472935:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573305.472941:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573305.472947:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573305.472964:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573305.472967:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573305.472970:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573305.472981:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573305.472983:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573305.472985:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573305.472987:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573305.473003:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573305.473005:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573305.473007:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573305.473009:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573305.473011:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573305.473036:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573305.473039:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573305.473040:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573305.473042:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573305.473044:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573305.473046:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573305.473047:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573305.473049:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573305.473051:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573305.473068:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573305.473070:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573305.473071:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573305.473074:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573305.473075:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573305.473077:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573305.473079:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573305.473080:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573305.473082:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573305.728905:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308665581, at 4308665600 00000100:00000010:0.0:1788573305.728920:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f00. 00000020:00000001:0.0:1788573306.177017:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573306.177020:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573306.177022:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573306.496941:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573306.496946:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573306.496950:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573306.496955:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573306.496961:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573306.496968:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573306.496974:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573306.496977:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573306.496983:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573307.521002:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573307.521007:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573307.521010:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573307.521014:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573307.521020:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573307.521024:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573307.521027:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573307.521032:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573307.521039:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573308.544969:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573308.544980:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573308.544986:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573308.544995:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573308.545009:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573308.545017:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573308.545032:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573308.545044:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573308.545052:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573308.545101:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573308.545105:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573308.545107:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573308.545125:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573308.545129:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573308.545132:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573308.545135:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573308.545138:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573308.545142:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573309.568940:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573309.568946:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573309.568950:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573309.568957:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573309.568964:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573309.568969:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573309.568985:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573309.568989:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573309.568998:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573310.592911:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573310.592919:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573310.592937:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573310.592942:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573310.592949:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573310.592954:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573310.592975:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573310.592977:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573310.592980:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573310.848911:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308670702, at 4308670720 00000100:00000010:0.0:1788573310.848929:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674780. 00000020:00000001:0.0:1788573311.297008:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573311.297012:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573311.297016:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573311.616916:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573311.616925:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573311.616929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573311.616938:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573311.616952:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573311.616958:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573311.616982:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573311.616986:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573311.616993:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573312.640928:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573312.640937:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573312.640943:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573312.640952:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573312.640961:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573312.640965:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573312.640979:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573312.640982:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573312.640985:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573313.664882:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573313.664890:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573313.664892:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573313.664906:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573313.664911:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573313.664917:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573313.664921:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573313.664932:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573313.664937:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573314.688951:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573314.688959:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573314.688963:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573314.688968:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573314.688974:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573314.688978:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573314.689005:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573314.689009:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573314.689011:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573315.712932:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573315.712938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573315.712941:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573315.712944:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573315.712948:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573315.712952:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573315.712974:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573315.712985:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573315.712988:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573315.968881:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308675821, at 4308675840 00000100:00000010:0.0:1788573315.968904:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674d80. 00000020:00000001:0.0:1788573316.416972:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573316.416976:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573316.416980:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573316.736944:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573316.736949:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573316.736951:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573316.736955:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573316.736958:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573316.736962:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573316.736975:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573316.736979:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573316.736986:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573317.760983:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573317.760989:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573317.760992:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573317.761023:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573317.761030:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573317.761048:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573317.761062:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573317.761069:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573317.761074:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573318.784956:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573318.784968:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573318.784977:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573318.785001:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573318.785008:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573318.785017:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573318.785034:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573318.785037:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573318.785043:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573320.832909:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573320.832918:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573320.832922:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573320.832933:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573320.832940:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573320.832947:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573320.832953:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573320.832956:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573320.832960:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573321.088932:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308680941, at 4308680960 00000100:00000010:0.0:1788573321.088948:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7de0. 00000020:00000001:0.0:1788573321.537079:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573321.537083:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573321.537087:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573321.857004:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573321.857015:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573321.857026:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573321.857039:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573321.857055:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573321.857067:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573321.857090:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573321.857093:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573321.857114:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573322.880932:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573322.880959:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573322.880965:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573322.880973:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573322.880978:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573322.880983:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573322.880989:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573322.880994:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573322.881003:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573322.881053:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573322.881060:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573322.881064:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573322.881074:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573322.881080:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573322.881084:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573322.881090:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573322.881093:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573322.881096:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573323.904949:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573323.904956:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573323.904961:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573323.904968:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573323.904977:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573323.904983:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573323.904992:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573323.905006:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573323.905013:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573324.928957:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573324.928968:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573324.928975:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573324.928983:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573324.928991:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573324.928995:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573324.929012:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573324.929017:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573324.929020:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573324.929063:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573324.929078:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573324.929083:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573324.929089:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573324.929096:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573324.929102:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573324.929106:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573324.929109:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573324.929112:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573325.952951:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573325.952956:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573325.952961:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573325.952966:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573325.952973:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573325.952981:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573325.953001:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573325.953006:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573325.953010:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573326.208961:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308686062, at 4308686080 00000100:00000010:0.0:1788573326.208990:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7720. 00000020:00000001:0.0:1788573326.657051:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573326.657055:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573326.657058:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573326.976971:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573326.976981:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573326.976987:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573326.976997:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573326.977005:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573326.977011:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573326.977055:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573326.977059:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573326.977062:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573327.488918:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573327.488932:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573327.488936:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573327.488942:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573327.488948:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573327.488982:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14014 00000400:00000200:0.0:1788573327.488993:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573327.488999:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573327.489002:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573327.489005:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489008:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573327.489010:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489012:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573327.489016:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573327.489018:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573327.489021:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573327.489025:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573327.489028:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573327.489033:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573327.489036:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489038:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573327.489041:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489043:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573327.489047:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573327.489052:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489065:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573327.489071:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573327.489075:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573327.489078:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489080:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573327.489082:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573327.489092:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573327.489095:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573327.489099:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573327.489101:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489104:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573327.489106:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573327.489116:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573327.489124:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573327.489131:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573327.489141:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516558592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573327.489151:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573327.489155:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573327.489158:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516558592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573327.489174:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573327.489177:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573327.489180:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573327.489181:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573327.489182:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573327.489184:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573327.489186:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14014 00000400:00000200:0.0:1788573327.489190:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573327.489193:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573327.489195:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573327.489198:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489200:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573327.489202:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489204:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573327.489206:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573327.489207:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573327.489210:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573327.489213:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573327.489214:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573327.489218:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573327.489221:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489227:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573327.489228:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489232:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573327.489234:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573327.489238:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489240:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573327.489241:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573327.489249:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573327.489254:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489258:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573327.489262:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573327.489267:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489270:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573327.489273:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573327.489276:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573327.489278:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573327.489280:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489283:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573327.489284:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573327.489287:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573327.489291:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573327.489294:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573327.489303:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516558720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573327.489311:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573327.489314:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573327.489318:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516558720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573327.489327:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573327.489330:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573327.489336:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573327.489342:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573327.489344:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573327.489348:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573327.489353:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14014 00000400:00000200:0.0:1788573327.489357:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573327.489362:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573327.489364:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573327.489366:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489368:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573327.489373:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489376:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573327.489378:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573327.489380:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573327.489382:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573327.489386:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573327.489391:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573327.489394:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573327.489401:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489405:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573327.489408:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489413:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573327.489418:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573327.489425:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489430:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573327.489433:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573327.489436:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573327.489440:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489443:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573327.489448:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573327.489453:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489463:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573327.489470:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573327.489475:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573327.489481:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573327.489485:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489488:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573327.489492:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573327.489496:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573327.489504:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573327.489509:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573327.489515:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516558848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573327.489524:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573327.489527:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573327.489533:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516558848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573327.489549:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573327.489553:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573327.489556:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573327.489557:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573327.489558:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573327.489561:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573327.489567:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14014 00000400:00000200:0.0:1788573327.489582:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573327.489587:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573327.489589:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573327.489592:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489596:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573327.489601:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489605:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573327.489608:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573327.489618:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573327.489621:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573327.489625:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573327.489630:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573327.489632:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573327.489637:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489644:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573327.489649:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489655:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573327.489660:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573327.489669:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489676:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573327.489681:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573327.489686:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573327.489689:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489690:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573327.489694:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573327.489697:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489701:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573327.489707:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573327.489710:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573327.489714:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573327.489716:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489718:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573327.489719:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573327.489721:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573327.489727:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573327.489732:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573327.489740:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516558976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573327.489748:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573327.489751:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573327.489764:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516558976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573327.489776:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573327.489782:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573327.489787:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573327.489789:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573327.489790:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573327.489794:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573327.489797:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14014 00000400:00000200:0.0:1788573327.489862:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573327.489871:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573327.489877:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573327.489895:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489899:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573327.489902:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489907:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573327.489909:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573327.489912:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573327.489914:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573327.489916:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573327.489918:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573327.489920:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573327.489923:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489924:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573327.489927:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489930:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573327.489936:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573327.489943:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573327.489949:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573327.489961:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573327.489967:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573327.489972:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489979:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573327.489983:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573327.489986:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.489989:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573327.489995:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573327.489999:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573327.490002:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573327.490006:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573327.490009:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573327.490011:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573327.490014:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573327.490021:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573327.490029:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573327.490040:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516559104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573327.490057:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573327.490063:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573327.490071:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516559104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573327.490083:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573327.490087:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14025) 00000100:00000001:0.0:1788573328.000932:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573328.000948:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573328.000953:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573328.000961:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573328.000967:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573328.000972:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573328.000987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573328.000991:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573328.000997:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573329.024986:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573329.025010:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573329.025015:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573329.025024:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573329.025038:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573329.025042:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573329.025071:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573329.025076:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573329.025083:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573330.048978:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573330.048986:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573330.048992:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573330.049011:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573330.049023:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573330.049032:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573330.049049:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573330.049054:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573330.049059:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573330.049084:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573330.049092:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573330.049094:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573330.049098:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573330.049103:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573330.049109:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573330.049113:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573330.049116:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573330.049119:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573331.072931:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573331.072938:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573331.072942:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573331.072948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573331.072954:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573331.072960:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573331.072974:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573331.072980:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573331.072984:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573331.328939:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308691181, at 4308691200 00000100:00000010:0.0:1788573331.328958:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea74e0. 00000020:00000001:0.0:1788573331.777081:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573331.777107:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573331.777110:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573332.096917:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573332.096923:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573332.096926:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573332.096933:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573332.096941:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573332.096945:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573332.096958:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573332.096961:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573332.096965:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573333.120966:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573333.120975:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573333.120978:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573333.120988:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573333.120995:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573333.121004:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573333.121015:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573333.121023:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573333.121035:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573333.121057:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573333.121061:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573333.121066:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573333.121075:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573333.121087:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573333.121097:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573333.121103:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573333.121106:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573333.121109:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573334.144939:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573334.144943:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573334.144947:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573334.144951:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573334.144957:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573334.144962:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573334.144968:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573334.144974:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573334.144980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573335.168975:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573335.168981:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573335.168984:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573335.168988:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573335.168994:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573335.168998:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573335.169003:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573335.169007:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573335.169012:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573336.192939:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573336.192945:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573336.192950:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573336.192957:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573336.192965:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573336.192970:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573336.192980:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573336.192982:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573336.192986:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573336.448922:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308696302, at 4308696320 00000100:00000010:0.0:1788573336.448944:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7720. 00000020:00000001:0.0:1788573336.897062:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573336.897066:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573336.897069:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573337.216946:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573337.216957:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573337.216961:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573337.216971:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573337.216978:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573337.216982:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573337.216988:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573337.216993:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573337.216998:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573337.728937:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573337.728950:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573337.728954:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573337.728958:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573337.728980:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573337.728988:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14025 00000400:00000200:0.0:1788573337.728998:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573337.729005:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573337.729008:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573337.729011:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729014:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573337.729017:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729019:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573337.729023:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573337.729026:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573337.729028:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573337.729032:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573337.729035:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573337.729037:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573337.729040:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729043:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573337.729044:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729047:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573337.729049:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573337.729053:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729057:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573337.729060:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573337.729063:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573337.729067:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.729069:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573337.729072:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573337.729079:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573337.729082:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573337.729087:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573337.729088:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.729098:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573337.729101:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573337.729106:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573337.729113:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573337.729119:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573337.729130:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516559872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573337.729140:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573337.729145:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573337.729151:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516559872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573337.729174:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573337.729177:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573337.729181:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573337.729184:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573337.729186:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573337.729190:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573337.729194:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14025 00000400:00000200:0.0:1788573337.729200:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573337.729206:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573337.729210:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573337.729217:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729225:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573337.729230:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729236:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573337.729242:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573337.729245:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573337.729250:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573337.729256:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573337.729260:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573337.729262:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573337.729278:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729283:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573337.729290:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729295:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573337.729300:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573337.729306:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729312:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573337.729314:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573337.729323:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573337.729330:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.729335:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573337.729341:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573337.729348:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.729353:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573337.729360:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573337.729363:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573337.729366:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573337.729368:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.729370:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573337.729372:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573337.729375:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573337.729380:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573337.729387:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573337.729395:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516560000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573337.729409:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573337.729414:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573337.729424:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516560000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573337.729449:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573337.729455:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573337.729467:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573337.729470:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573337.729472:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573337.729476:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573337.729482:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14025 00000400:00000200:0.0:1788573337.729490:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573337.729497:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573337.729501:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573337.729509:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729512:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573337.729517:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729521:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573337.729526:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573337.729529:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573337.729532:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573337.729535:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573337.729540:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573337.729543:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573337.729550:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729553:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573337.729560:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729565:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573337.729569:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573337.729574:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729581:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573337.729584:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573337.729590:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573337.729596:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.729601:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573337.729605:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573337.729618:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.729623:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573337.729630:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573337.729636:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573337.729642:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573337.729645:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.729648:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573337.729651:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573337.729655:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573337.729660:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573337.729666:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573337.729677:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516560128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573337.729694:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573337.729699:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573337.729709:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516560128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573337.729724:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573337.729729:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573337.729737:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573337.729742:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573337.729746:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573337.729752:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573337.729758:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14025 00000400:00000200:0.0:1788573337.729768:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573337.729775:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573337.729779:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573337.729785:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729789:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573337.729795:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729847:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573337.729853:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573337.729858:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573337.729862:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573337.729866:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573337.729886:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573337.729893:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573337.729903:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729910:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573337.729917:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729920:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573337.729924:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573337.729930:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573337.729937:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573337.729942:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573337.729946:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573337.729952:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.729956:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573337.729961:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573337.729965:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.729970:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573337.729977:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573337.729981:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573337.729988:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573337.729992:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.730002:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573337.730005:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573337.730010:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573337.730019:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573337.730027:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573337.730037:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516560256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573337.730074:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573337.730078:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573337.730086:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516560256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573337.730106:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573337.730110:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573337.730120:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573337.730123:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573337.730128:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573337.730136:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573337.730143:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14025 00000400:00000200:0.0:1788573337.730155:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573337.730166:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573337.730172:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573337.730182:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.730189:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573337.730197:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.730203:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573337.730209:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573337.730214:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573337.730219:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573337.730223:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573337.730229:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573337.730234:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573337.730243:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.730247:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573337.730252:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573337.730257:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573337.730268:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573337.730277:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573337.730283:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573337.730288:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573337.730293:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573337.730300:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.730305:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573337.730310:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573337.730317:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.730322:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573337.730329:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573337.730335:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573337.730342:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573337.730346:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573337.730353:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573337.730358:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573337.730363:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573337.730371:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573337.730380:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573337.730390:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516560384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573337.730403:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573337.730408:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573337.730416:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516560384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573337.730442:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573337.730447:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14035) 00000100:00000001:0.0:1788573338.240934:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573338.240942:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573338.240946:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573338.240954:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573338.240959:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573338.240973:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573338.240977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573338.240981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573338.240984:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573339.265076:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573339.265081:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573339.265087:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573339.265091:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573339.265099:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573339.265103:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573339.265107:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573339.265109:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573339.265111:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573340.289053:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573340.289064:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573340.289070:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573340.289078:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573340.289089:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573340.289098:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573340.289103:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573340.289106:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573340.289110:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573341.312989:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573341.312995:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573341.312999:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573341.313006:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573341.313012:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573341.313016:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573341.313026:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573341.313028:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573341.313031:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573341.568928:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308701421, at 4308701440 00000100:00000010:0.0:1788573341.568946:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea71e0. 00000020:00000001:0.0:1788573342.017028:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573342.017031:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573342.017033:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788573342.849285:0:2391:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788573342.849307:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573342.849327:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573342.849336:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573342.849345:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573345.408983:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573345.408987:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573345.408991:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573345.408995:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573345.409000:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573345.409005:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573345.409010:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573345.409015:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573345.409020:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573346.432933:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573346.432938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573346.432942:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573346.432946:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573346.432954:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573346.432960:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573346.432974:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573346.432978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573346.432983:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573346.688904:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308706542, at 4308706560 00000100:00000010:0.0:1788573346.688919:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7840. 00000020:00000001:0.0:1788573347.136965:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573347.136969:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573347.136973:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573347.456997:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573347.457011:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573347.457019:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573347.457036:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573347.457057:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573347.457067:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573347.457077:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573347.457082:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573347.457087:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573348.480952:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573348.480984:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573348.480989:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573348.481001:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573348.481010:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573348.481014:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573348.481018:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573348.481026:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573348.481030:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573349.504939:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573349.504945:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573349.504949:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573349.504958:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573349.504967:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573349.504974:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573349.504977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573349.504982:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573349.504988:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573349.505008:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573349.505014:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573349.505019:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573349.505024:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573349.505029:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573349.505034:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573349.505037:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573349.505039:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573349.505041:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573350.528975:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573350.528982:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573350.528987:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573350.528995:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573350.529001:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573350.529007:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573350.529020:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573350.529024:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573350.529028:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573351.552920:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573351.552928:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573351.552932:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573351.552952:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573351.552958:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573351.552963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573351.552984:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573351.552988:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573351.552991:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573351.808920:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308711661, at 4308711680 00000100:00000010:0.0:1788573351.808937:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7d80. 00000020:00000001:0.0:1788573352.257016:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573352.257019:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573352.257022:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573352.576919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573352.576928:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573352.576934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573352.576951:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573352.576957:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573352.576963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573352.576981:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573352.576986:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573352.576990:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573353.600923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573353.600933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573353.600937:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573353.600946:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573353.600955:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573353.600962:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573353.600983:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573353.600988:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573353.600994:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573354.624953:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573354.624970:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573354.624974:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573354.624979:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573354.624987:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573354.624993:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573354.624999:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573354.625003:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573354.625026:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573356.672968:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573356.672976:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573356.672988:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573356.673006:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573356.673026:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573356.673035:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573356.673048:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573356.673052:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573356.673061:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573356.928933:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308716781, at 4308716800 00000100:00000010:0.0:1788573356.928959:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c00. 00000020:00000001:0.0:1788573357.377087:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573357.377093:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573357.377096:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573357.696948:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573357.696960:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573357.696964:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573357.696972:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573357.696981:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573357.696989:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573357.696997:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573357.697002:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573357.697007:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573357.697059:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573357.697066:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573357.697069:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573357.697073:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573357.697077:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573357.697081:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573357.697085:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573357.697087:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573357.697091:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573358.720970:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573358.720982:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573358.720991:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573358.721003:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573358.721030:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573358.721045:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573358.721084:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573358.721088:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573358.721092:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573359.744964:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573359.744969:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573359.744972:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573359.744979:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573359.744985:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573359.744993:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573359.744996:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573359.745004:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573359.745010:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573359.745046:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573359.745059:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573359.745063:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573359.745072:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573359.745089:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573359.745103:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573359.745115:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573359.745121:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573359.745127:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573360.768986:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573360.768991:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573360.768997:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573360.769006:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573360.769016:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573360.769023:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573360.769045:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573360.769048:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573360.769053:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573360.769073:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573360.769076:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573360.769078:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573360.769079:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573360.769081:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573360.769083:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573360.769101:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573360.769103:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573360.769107:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573361.792952:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573361.792961:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573361.792967:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573361.792972:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573361.792979:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573361.792984:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573361.792987:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573361.792990:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573361.792994:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573362.048936:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308721902, at 4308721920 00000100:00000010:0.0:1788573362.048958:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ea0. 00000020:00000001:0.0:1788573362.497016:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573362.497021:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573362.497024:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573362.816993:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573362.817001:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573362.817008:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573362.817016:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573362.817025:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573362.817030:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573362.817034:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573362.817038:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573362.817043:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573363.840898:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573363.840906:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573363.840908:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573363.840916:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573363.840924:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573363.840930:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573363.840935:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573363.840940:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573363.840947:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573363.840990:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573363.840994:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573363.841014:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573363.841016:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573363.841018:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573363.841020:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573363.841021:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573363.841022:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573363.841024:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573364.864965:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573364.864971:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573364.864976:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573364.864985:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573364.864991:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573364.864996:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573364.864999:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573364.865001:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573364.865002:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573365.888929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573365.888938:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573365.888941:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573365.888948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573365.888955:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573365.888961:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573365.888971:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573365.888977:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573365.888984:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573365.889021:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573365.889026:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573365.889027:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573365.889031:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573365.889034:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573365.889036:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573365.889038:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573365.889040:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573365.889042:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573366.912963:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573366.912969:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573366.912974:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573366.912982:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573366.913004:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573366.913008:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573366.913011:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573366.913013:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573366.913017:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573367.168971:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308727021, at 4308727040 00000100:00000010:0.0:1788573367.168991:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ba0. 00000020:00000001:0.0:1788573367.617017:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573367.617022:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573367.617026:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573367.936909:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573367.936919:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573367.936923:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573367.936931:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573367.936938:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573367.936943:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573367.936957:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573367.936962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573367.936967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573368.960911:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573368.960918:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573368.960922:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573368.960932:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573368.960941:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573368.960950:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573368.960983:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573368.960987:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573368.960990:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573369.984923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573369.984928:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573369.984932:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573369.984937:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573369.984941:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573369.984946:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573369.984971:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573369.984975:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573369.984980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573371.008993:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573371.009003:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573371.009011:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573371.009021:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573371.009032:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573371.009041:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573371.009073:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573371.009077:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573371.009082:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573372.032911:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573372.032919:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573372.032922:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573372.032931:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573372.032938:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573372.032943:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573372.032950:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573372.032963:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573372.032967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573372.032994:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573372.032998:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573372.033000:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573372.033002:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573372.033005:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573372.033008:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573372.033014:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573372.033016:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573372.033018:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573372.288913:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308732141, at 4308732160 00000100:00000010:0.0:1788573372.288932:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7b40. 00000020:00000001:0.0:1788573372.737036:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573372.737039:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573372.737042:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573373.057148:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573373.057151:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573373.057152:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573373.057155:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573373.057156:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573373.057170:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573373.057171:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573373.057172:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573373.057174:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573374.080910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573374.080921:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573374.080926:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573374.080933:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573374.080940:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573374.080945:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573374.080964:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573374.080968:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573374.080972:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573375.104934:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573375.104943:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573375.104948:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573375.104957:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573375.104963:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573375.104973:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573375.104982:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573375.104988:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573375.105000:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573376.128913:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573376.128918:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573376.128923:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573376.128931:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573376.128947:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573376.128951:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573376.128961:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573376.128965:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573376.128971:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573377.152938:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573377.152945:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573377.152948:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573377.152957:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573377.152962:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573377.152968:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573377.152974:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573377.152986:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573377.152990:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573377.408913:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308737262, at 4308737280 00000100:00000010:0.0:1788573377.408931:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7420. 00000020:00000001:0.0:1788573377.856999:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573377.857003:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573377.857006:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573378.176965:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573378.176976:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573378.176982:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573378.176990:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573378.176997:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573378.177003:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573378.177007:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573378.177011:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573378.177014:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573379.200943:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573379.200950:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573379.200956:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573379.200965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573379.200975:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573379.200981:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573379.200998:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573379.201003:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573379.201010:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573380.224939:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573380.224947:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573380.224954:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573380.224965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573380.224974:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573380.224980:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573380.224988:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573380.224992:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573380.224998:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573380.737143:0:2391:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788573380.737149:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573380.737163:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573380.737181:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573380.737185:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573381.248912:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573381.248922:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573381.248927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573381.248934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573381.248942:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573381.248949:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573381.248962:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573381.248965:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573381.248971:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573382.272949:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573382.272956:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573382.272960:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573382.272965:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573382.272972:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573382.272977:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573382.273120:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573382.273123:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573382.273126:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573382.528945:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308742381, at 4308742400 00000100:00000010:0.0:1788573382.528970:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea75a0. 00000020:00000001:0.0:1788573382.977033:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573382.977036:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573382.977038:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573383.297028:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573383.297035:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573383.297040:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573383.297045:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573383.297049:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573383.297057:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573383.297064:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573383.297069:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573383.297080:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573384.320984:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573384.320994:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573384.320999:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573384.321027:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573384.321043:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573384.321051:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573384.321059:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573384.321064:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573384.321075:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573385.344992:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573385.345006:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573385.345017:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573385.345031:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573385.345038:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573385.345054:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573385.345069:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573385.345073:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573385.345076:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573386.368948:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573386.368954:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573386.368959:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573386.368965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573386.368971:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573386.368976:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573386.368982:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573386.368986:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573386.368991:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573387.392970:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573387.392978:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573387.392989:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573387.392996:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573387.393002:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573387.393014:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573387.393040:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573387.393043:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573387.393046:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573387.648958:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308747502, at 4308747520 00000100:00000010:0.0:1788573387.648977:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7540. 00000020:00000001:0.0:1788573388.096985:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573388.096988:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573388.097010:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573388.416959:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573388.416966:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573388.416972:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573388.416979:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573388.416987:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573388.416993:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573388.417010:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573388.417014:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573388.417018:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573389.440931:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573389.440938:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573389.440941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573389.440948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573389.440952:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573389.440957:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573389.440962:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573389.440966:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573389.440971:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573390.464987:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573390.465000:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573390.465013:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573390.465022:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573390.465032:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573390.465046:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573390.465076:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573390.465083:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573390.465094:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573391.488977:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573391.488986:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573391.489001:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573391.489010:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573391.489015:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573391.489023:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573391.489028:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573391.489031:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573391.489035:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573392.512948:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573392.512968:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573392.512971:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573392.512976:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573392.512982:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573392.512986:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573392.512996:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573392.512999:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573392.513001:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573392.768943:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308752621, at 4308752640 00000100:00000010:0.0:1788573392.768961:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea77e0. 00000020:00000001:0.0:1788573393.217079:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573393.217084:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573393.217088:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573393.536979:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573393.536987:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573393.536992:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573393.537000:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573393.537017:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573393.537024:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573393.537035:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573393.537038:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573393.537045:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573394.048949:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573394.048963:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573394.048966:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573394.048971:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573394.048976:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573394.048983:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14081 00000400:00000200:0.0:1788573394.048991:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573394.048997:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573394.049000:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573394.049003:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049005:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573394.049007:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049034:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573394.049038:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573394.049039:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573394.049042:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573394.049044:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573394.049046:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573394.049047:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573394.049050:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049051:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573394.049052:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049053:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573394.049055:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573394.049057:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049059:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573394.049061:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573394.049064:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573394.049066:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.049068:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573394.049069:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573394.049074:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573394.049077:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573394.049080:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573394.049082:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.049084:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573394.049086:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573394.049090:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573394.049096:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573394.049102:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573394.049109:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516566912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573394.049118:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573394.049121:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573394.049129:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516566912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573394.049152:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573394.049154:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573394.049157:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573394.049159:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573394.049161:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573394.049163:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573394.049165:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14081 00000400:00000200:0.0:1788573394.049169:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573394.049175:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573394.049179:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573394.049184:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049187:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573394.049191:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049195:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573394.049200:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573394.049201:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573394.049204:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573394.049207:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573394.049211:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573394.049214:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573394.049218:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049221:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573394.049225:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049231:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573394.049237:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573394.049246:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049252:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573394.049263:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573394.049272:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573394.049285:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.049287:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573394.049293:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573394.049297:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.049301:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573394.049307:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573394.049310:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573394.049312:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573394.049314:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.049317:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573394.049321:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573394.049325:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573394.049332:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573394.049339:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573394.049346:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516567040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573394.049364:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573394.049370:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573394.049378:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516567040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573394.049395:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573394.049399:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573394.049405:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573394.049409:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573394.049410:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573394.049413:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573394.049418:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14081 00000400:00000200:0.0:1788573394.049426:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573394.049431:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573394.049435:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573394.049444:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049448:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573394.049453:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049457:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573394.049462:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573394.049464:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573394.049466:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573394.049470:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573394.049475:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573394.049477:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573394.049481:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049486:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573394.049492:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049497:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573394.049504:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573394.049510:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049517:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573394.049532:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573394.049536:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573394.049541:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.049544:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573394.049547:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573394.049551:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.049553:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573394.049556:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573394.049560:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573394.049566:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573394.049568:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.049572:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573394.049573:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573394.049576:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573394.049579:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573394.049592:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573394.049600:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516567168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573394.049610:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573394.049614:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573394.049622:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516567168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573394.049633:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573394.049638:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573394.049641:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573394.049642:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573394.049643:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573394.049645:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573394.049649:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14081 00000400:00000200:0.0:1788573394.049657:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573394.049661:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573394.049666:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573394.049668:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049670:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573394.049671:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049673:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573394.049675:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573394.049677:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573394.049681:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573394.049682:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573394.049684:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573394.049686:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573394.049689:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049693:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573394.049702:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049705:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573394.049710:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573394.049714:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049718:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573394.049720:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573394.049724:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573394.049727:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.049732:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573394.049735:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573394.049738:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.049739:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573394.049744:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573394.049746:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573394.049750:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573394.049752:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.049756:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573394.049759:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573394.049761:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573394.049766:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573394.049771:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573394.049781:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516567296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573394.049841:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573394.049845:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573394.049852:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516567296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573394.049864:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573394.049867:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573394.049873:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573394.049890:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573394.049898:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573394.049904:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573394.049911:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14081 00000400:00000200:0.0:1788573394.049922:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573394.049931:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573394.049937:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573394.049947:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049953:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573394.049960:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.049965:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573394.049972:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573394.049976:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573394.049979:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573394.049983:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573394.049987:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573394.049992:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573394.049997:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.050000:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573394.050005:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573394.050010:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573394.050012:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573394.050015:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573394.050016:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573394.050019:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573394.050022:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573394.050027:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.050030:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573394.050033:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573394.050038:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.050040:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573394.050049:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573394.050053:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573394.050056:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573394.050060:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573394.050061:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573394.050065:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573394.050067:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573394.050071:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573394.050076:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573394.050085:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516567424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573394.050103:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573394.050107:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573394.050114:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516567424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573394.050129:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573394.050134:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14091) 00000100:00000001:0.0:1788573394.560948:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573394.560957:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573394.560960:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573394.560969:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573394.560977:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573394.560981:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573394.560995:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573394.560998:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573394.561001:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573395.584957:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573395.584960:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573395.584964:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573395.584971:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573395.584978:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573395.584983:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573395.584995:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573395.584999:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573395.585003:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573396.608961:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573396.608967:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573396.608971:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573396.608977:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573396.608982:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573396.608988:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573396.608994:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573396.608997:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573396.609000:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573397.633004:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573397.633006:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573397.633008:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573397.633011:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573397.633013:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573397.633015:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573397.633017:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573397.633019:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573397.633021:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573397.633114:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573397.633117:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573397.633118:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573397.633121:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573397.633123:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573397.633126:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573397.633127:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573397.633129:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573397.633131:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573397.888993:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308757741, at 4308757760 00000100:00000010:0.0:1788573397.888998:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f60. 00000020:00000001:0.0:1788573398.336998:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573398.337008:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573398.337011:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573398.656915:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573398.656926:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573398.656929:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573398.656937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573398.656945:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573398.656976:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573398.656985:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573398.656989:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573398.656995:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573398.657036:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573398.657039:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573398.657040:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573398.657042:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573398.657044:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573398.657046:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573398.657048:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573398.657049:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573398.657051:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573399.680920:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573399.680930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573399.680935:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573399.680945:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573399.680954:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573399.680959:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573399.680964:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573399.680971:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573399.680988:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573400.704916:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573400.704922:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573400.704925:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573400.704933:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573400.704938:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573400.704942:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573400.704951:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573400.704956:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573400.704960:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573401.728952:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573401.728957:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573401.728962:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573401.728970:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573401.728978:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573401.728984:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573401.728993:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573401.729028:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573401.729032:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573402.752967:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573402.752973:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573402.752978:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573402.752986:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573402.752992:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573402.752997:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573402.753000:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573402.753003:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573402.753007:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573403.008905:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308762862, at 4308762880 00000100:00000010:0.0:1788573403.008924:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea79c0. 00000020:00000001:0.0:1788573403.456955:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573403.456958:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573403.456961:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573403.776920:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573403.776928:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573403.776934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573403.776940:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573403.776947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573403.776954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573403.776972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573403.776976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573403.776980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573404.800937:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573404.800947:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573404.800953:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573404.800962:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573404.800969:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573404.800976:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573404.800993:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573404.800997:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573404.801000:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573405.824944:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573405.824951:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573405.824956:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573405.824977:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573405.824985:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573405.824989:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573405.825000:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573405.825003:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573405.825007:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573405.825027:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573405.825030:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573405.825033:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573405.825038:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573405.825044:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573405.825049:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573405.825054:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573405.825059:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573405.825065:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573406.848961:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573406.848967:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573406.848972:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573406.848979:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573406.848986:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573406.848992:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573406.849000:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573406.849012:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573406.849018:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573407.872926:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573407.872935:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573407.872938:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573407.872946:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573407.872952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573407.872958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573407.872963:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573407.872968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573407.872977:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573407.873020:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573407.873024:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573407.873027:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573407.873030:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573407.873033:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573407.873042:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573407.873043:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573407.873044:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573407.873046:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573408.128919:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308767981, at 4308768000 00000100:00000010:0.0:1788573408.128935:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7a80. 00000020:00000001:0.0:1788573408.577051:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573408.577056:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573408.577060:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573408.896928:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573408.896938:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573408.896942:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573408.896962:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573408.896970:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573408.896975:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573408.897106:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573408.897109:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573408.897112:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573409.920920:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573409.920929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573409.920934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573409.920941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573409.920945:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573409.920950:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573409.920956:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573409.920959:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573409.920962:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573410.944974:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573410.944979:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573410.944981:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573410.944986:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573410.944990:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573410.944993:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573410.944995:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573410.944998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573410.945001:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573411.968905:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573411.968931:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573411.968936:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573411.968944:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573411.968950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573411.968958:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573411.968965:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573411.968970:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573411.968974:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573411.969003:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573411.969007:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573411.969008:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573411.969010:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573411.969012:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573411.969015:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573411.969017:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573411.969018:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573411.969021:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573412.992906:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573412.992912:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573412.992916:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573412.992924:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573412.992930:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573412.992936:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573412.992952:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573412.992956:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573412.992960:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573412.992979:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573412.992984:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573412.992987:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573412.992994:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573412.992998:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573412.993002:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573412.993007:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573412.993010:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573412.993014:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573413.248898:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308773101, at 4308773120 00000100:00000010:0.0:1788573413.248916:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7420. 00000020:00000001:0.0:1788573413.697047:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573413.697069:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573413.697073:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573414.016971:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573414.016986:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573414.016993:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573414.016998:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573414.017008:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573414.017017:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573414.017026:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573414.017031:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573414.017044:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573414.017074:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573414.017078:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573414.017081:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573414.017084:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573414.017089:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573414.017092:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573414.017095:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573414.017097:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573414.017100:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573415.040972:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573415.040979:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573415.040984:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573415.040990:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573415.040996:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573415.041001:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573415.041030:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573415.041040:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573415.041058:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573416.064977:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573416.064984:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573416.064986:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573416.064992:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573416.064997:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573416.065002:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573416.065008:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573416.065013:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573416.065025:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573417.088974:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573417.088984:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573417.088990:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573417.088998:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573417.089006:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573417.089012:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573417.089031:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573417.089037:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573417.089052:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573418.112931:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573418.112938:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573418.112942:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573418.112949:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573418.112953:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573418.112959:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573418.112965:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573418.112968:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573418.112972:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573418.368916:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308778221, at 4308778240 00000100:00000010:0.0:1788573418.368933:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7b40. 00000020:00000001:0.0:1788573418.817018:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573418.817021:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573418.817024:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573419.136971:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573419.136979:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573419.136982:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573419.136988:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573419.136997:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573419.137005:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573419.137011:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573419.137022:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573419.137041:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573419.648917:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573419.648925:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573419.648929:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573419.648933:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573419.648959:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573419.648963:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14107 00000400:00000200:0.0:1788573419.648973:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573419.648981:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573419.648985:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573419.648990:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.648993:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573419.648996:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.648999:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573419.649004:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573419.649006:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573419.649017:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573419.649021:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573419.649024:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573419.649027:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573419.649031:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649033:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573419.649036:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649038:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573419.649042:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573419.649045:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649048:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573419.649052:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573419.649055:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573419.649059:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.649062:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573419.649064:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573419.649072:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573419.649076:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573419.649081:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573419.649083:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.649099:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573419.649102:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573419.649108:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573419.649116:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573419.649123:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573419.649134:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516570112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573419.649149:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573419.649155:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573419.649164:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516570112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573419.649187:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573419.649191:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573419.649195:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573419.649197:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573419.649199:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573419.649202:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573419.649206:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14107 00000400:00000200:0.0:1788573419.649211:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573419.649216:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573419.649219:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573419.649225:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649229:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573419.649233:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649244:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573419.649252:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573419.649255:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573419.649258:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573419.649263:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573419.649269:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573419.649275:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573419.649291:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649295:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573419.649297:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649300:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573419.649304:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573419.649309:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649315:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573419.649321:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573419.649329:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573419.649333:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.649339:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573419.649376:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573419.649388:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.649392:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573419.649400:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573419.649406:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573419.649412:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573419.649415:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.649420:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573419.649424:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573419.649430:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573419.649438:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573419.649444:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573419.649454:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516570240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573419.649470:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573419.649474:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573419.649482:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516570240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573419.649502:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573419.649508:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573419.649524:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573419.649528:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573419.649531:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573419.649535:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573419.649542:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14107 00000400:00000200:0.0:1788573419.649560:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573419.649569:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573419.649573:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573419.649582:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649586:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573419.649591:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649596:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573419.649599:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573419.649601:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573419.649604:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573419.649607:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573419.649610:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573419.649612:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573419.649616:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649620:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573419.649625:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649627:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573419.649630:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573419.649634:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649639:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573419.649641:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573419.649646:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573419.649650:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.649654:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573419.649658:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573419.649675:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.649680:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573419.649687:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573419.649694:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573419.649698:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573419.649699:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.649702:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573419.649705:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573419.649710:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573419.649716:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573419.649721:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573419.649729:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516570368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573419.649741:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573419.649745:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573419.649751:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516570368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573419.649762:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573419.649765:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573419.649768:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573419.649772:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573419.649774:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573419.649777:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573419.649782:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14107 00000400:00000200:0.0:1788573419.649788:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573419.649793:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573419.649796:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573419.649799:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649803:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573419.649831:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649839:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573419.649845:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573419.649848:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573419.649852:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573419.649857:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573419.649864:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573419.649868:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573419.649877:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649882:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573419.649888:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649893:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573419.649896:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573419.649900:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573419.649903:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573419.649906:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573419.649911:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573419.649915:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.649919:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573419.649923:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573419.649927:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.649929:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573419.649935:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573419.649940:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573419.649947:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573419.649948:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.649952:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573419.649954:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573419.649957:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573419.649962:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573419.649965:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573419.649970:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516570496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573419.650009:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573419.650014:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573419.650021:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516570496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573419.650050:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573419.650057:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573419.650065:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573419.650071:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573419.650076:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573419.650085:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573419.650093:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14107 00000400:00000200:0.0:1788573419.650101:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573419.650108:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573419.650111:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573419.650116:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.650121:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573419.650126:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.650130:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573419.650133:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573419.650135:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573419.650137:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573419.650141:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573419.650146:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573419.650148:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573419.650152:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.650155:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573419.650160:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573419.650163:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573419.650173:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573419.650179:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573419.650181:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573419.650183:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573419.650186:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573419.650193:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.650200:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573419.650205:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573419.650212:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.650216:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573419.650223:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573419.650230:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573419.650237:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573419.650241:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573419.650244:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573419.650248:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573419.650252:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573419.650260:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573419.650268:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573419.650276:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516570624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573419.650295:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573419.650299:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573419.650310:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516570624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573419.650329:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573419.650337:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14117) 00000100:00000001:0.0:1788573420.160917:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573420.160926:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573420.160930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573420.160938:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573420.160944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573420.160960:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573420.160980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573420.160984:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573420.160991:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573421.184964:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573421.184975:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573421.184979:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573421.184989:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573421.184997:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573421.185003:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573421.185017:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573421.185021:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573421.185027:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573421.185086:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573421.185090:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573421.185093:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573421.185100:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573421.185103:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573421.185106:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573421.185117:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573421.185122:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573421.185124:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573422.208922:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573422.208927:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573422.208931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573422.208938:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573422.208942:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573422.208946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573422.208952:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573422.208954:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573422.208961:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573423.488891:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308783341, at 4308783360 00000100:00000010:0.0:1788573423.488903:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7360. 00000020:00000001:0.0:1788573423.937041:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573423.937046:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573423.937050:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573424.256919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573424.256939:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573424.256944:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573424.256951:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573424.256955:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573424.256960:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573424.256975:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573424.256978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573424.256982:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573424.768895:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573424.768908:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573424.768911:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573424.768915:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573424.768921:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573424.768930:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14112 00000400:00000200:0.0:1788573424.768941:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573424.768949:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573424.768952:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573424.768956:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.768959:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573424.768962:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.768965:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573424.768969:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573424.768971:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573424.768974:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573424.768976:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573424.768979:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573424.768981:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573424.768985:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.768986:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573424.768988:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.768990:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573424.768993:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573424.769016:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769019:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573424.769023:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573424.769026:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573424.769029:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.769031:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573424.769034:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573424.769040:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573424.769044:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573424.769048:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573424.769050:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.769054:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573424.769056:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573424.769060:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573424.769069:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573424.769076:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573424.769086:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516570752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573424.769098:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573424.769103:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573424.769108:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516570752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573424.769133:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573424.769137:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573424.769141:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573424.769144:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573424.769146:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573424.769150:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573424.769153:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14112 00000400:00000200:0.0:1788573424.769160:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573424.769168:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573424.769173:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573424.769189:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769194:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573424.769200:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769207:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573424.769211:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573424.769215:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573424.769220:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573424.769224:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573424.769230:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573424.769235:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573424.769243:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769250:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573424.769256:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769261:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573424.769266:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573424.769275:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769278:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573424.769280:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573424.769287:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573424.769294:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.769300:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573424.769306:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573424.769313:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.769317:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573424.769324:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573424.769329:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573424.769336:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573424.769339:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.769343:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573424.769345:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573424.769348:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573424.769356:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573424.769371:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573424.769383:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516570880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573424.769408:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573424.769413:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573424.769422:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516570880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573424.769478:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573424.769485:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573424.769492:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573424.769498:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573424.769502:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573424.769509:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573424.769516:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14112 00000400:00000200:0.0:1788573424.769526:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573424.769531:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573424.769533:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573424.769539:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769545:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573424.769550:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769555:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573424.769558:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573424.769562:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573424.769564:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573424.769567:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573424.769570:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573424.769573:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573424.769579:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769583:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573424.769597:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769604:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573424.769612:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573424.769620:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769627:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573424.769632:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573424.769638:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573424.769644:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.769649:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573424.769655:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573424.769661:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.769666:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573424.769672:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573424.769678:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573424.769686:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573424.769689:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.769695:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573424.769699:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573424.769703:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573424.769710:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573424.769718:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573424.769730:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516571008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573424.769747:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573424.769754:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573424.769769:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516571008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573424.769792:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573424.769799:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573424.769839:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573424.769844:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573424.769861:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573424.769868:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573424.769876:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14112 00000400:00000200:0.0:1788573424.769885:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573424.769892:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573424.769899:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573424.769906:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769914:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573424.769922:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769927:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573424.769935:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573424.769939:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573424.769944:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573424.769950:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573424.769958:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573424.769963:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573424.769971:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769977:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573424.769983:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.769990:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573424.769996:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573424.770007:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573424.770014:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573424.770019:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573424.770026:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573424.770035:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.770040:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573424.770046:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573424.770052:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.770057:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573424.770076:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573424.770082:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573424.770089:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573424.770093:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.770097:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573424.770101:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573424.770105:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573424.770112:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573424.770119:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573424.770130:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516571136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573424.770144:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573424.770149:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573424.770159:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516571136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573424.770174:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573424.770180:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573424.770189:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573424.770192:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573424.770196:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573424.770203:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573424.770208:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14112 00000400:00000200:0.0:1788573424.770216:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573424.770224:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573424.770229:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573424.770236:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.770242:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573424.770247:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.770251:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573424.770255:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573424.770266:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573424.770270:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573424.770274:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573424.770279:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573424.770284:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573424.770294:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.770315:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573424.770322:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573424.770329:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573424.770336:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573424.770346:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573424.770354:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573424.770358:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573424.770365:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573424.770373:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.770377:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573424.770381:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573424.770387:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.770394:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573424.770401:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573424.770407:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573424.770413:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573424.770417:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573424.770423:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573424.770427:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573424.770432:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573424.770455:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573424.770463:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573424.770486:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516571264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573424.770502:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573424.770508:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573424.770525:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516571264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573424.770548:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573424.770555:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14122) 00000100:00000001:0.0:1788573425.280924:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573425.280933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573425.280936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573425.280945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573425.280952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573425.280958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573425.280964:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573425.280968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573425.280975:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573426.304905:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573426.304910:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573426.304915:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573426.304923:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573426.304929:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573426.304933:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573426.304943:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573426.304946:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573426.304948:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573427.328948:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573427.328958:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573427.328965:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573427.328984:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573427.328990:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573427.328994:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573427.329016:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573427.329020:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573427.329023:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573428.352942:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573428.352948:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573428.352954:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573428.352961:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573428.352970:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573428.352976:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573428.353005:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573428.353011:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573428.353019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573428.608915:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308788461, at 4308788480 00000100:00000010:0.0:1788573428.608933:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ea0. 00000020:00000001:0.0:1788573429.057062:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573429.057070:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573429.057075:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573429.376982:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573429.376989:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573429.376998:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573429.377013:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573429.377017:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573429.377023:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573429.377040:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573429.377047:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573429.377052:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573429.888954:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573429.888968:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573429.888972:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573429.888977:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573429.888982:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573429.888990:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14117 00000400:00000200:0.0:1788573429.888999:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573429.889007:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573429.889009:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573429.889012:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889015:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573429.889017:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889019:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573429.889022:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573429.889024:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573429.889027:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573429.889050:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573429.889053:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573429.889056:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573429.889060:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889062:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573429.889065:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889067:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573429.889070:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573429.889074:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889076:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573429.889078:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573429.889081:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573429.889083:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.889085:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573429.889087:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573429.889093:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573429.889096:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573429.889099:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573429.889100:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.889103:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573429.889105:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573429.889109:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573429.889115:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573429.889121:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573429.889128:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516571392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573429.889136:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573429.889140:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573429.889143:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516571392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573429.889167:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573429.889170:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573429.889177:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573429.889178:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573429.889179:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573429.889181:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573429.889183:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14117 00000400:00000200:0.0:1788573429.889186:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573429.889189:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573429.889192:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573429.889195:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889197:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573429.889199:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889201:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573429.889202:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573429.889203:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573429.889204:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573429.889208:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573429.889210:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573429.889214:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573429.889219:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889221:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573429.889227:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889229:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573429.889237:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573429.889239:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889241:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573429.889242:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573429.889248:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573429.889252:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.889258:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573429.889262:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573429.889276:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.889283:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573429.889287:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573429.889291:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573429.889293:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573429.889294:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.889295:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573429.889296:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573429.889298:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573429.889301:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573429.889303:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573429.889311:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516571520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573429.889318:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573429.889322:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573429.889326:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516571520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573429.889338:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573429.889342:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573429.889347:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573429.889352:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573429.889355:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573429.889361:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573429.889366:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14117 00000400:00000200:0.0:1788573429.889371:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573429.889381:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573429.889389:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573429.889395:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889398:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573429.889399:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889410:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573429.889414:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573429.889415:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573429.889417:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573429.889418:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573429.889422:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573429.889425:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573429.889433:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889435:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573429.889437:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889439:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573429.889444:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573429.889447:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889450:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573429.889451:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573429.889453:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573429.889457:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.889463:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573429.889465:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573429.889467:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.889468:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573429.889470:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573429.889480:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573429.889485:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573429.889487:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.889489:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573429.889491:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573429.889493:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573429.889499:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573429.889508:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573429.889520:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516571648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573429.889571:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573429.889580:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573429.889588:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516571648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573429.889609:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573429.889619:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573429.889624:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573429.889626:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573429.889628:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573429.889631:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573429.889638:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14117 00000400:00000200:0.0:1788573429.889645:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573429.889653:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573429.889656:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573429.889664:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889669:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573429.889676:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889683:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573429.889691:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573429.889695:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573429.889700:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573429.889705:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573429.889715:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573429.889718:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573429.889731:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889741:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573429.889748:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889754:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573429.889765:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573429.889775:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573429.889779:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573429.889787:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573429.889791:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573429.889799:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.889841:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573429.889848:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573429.889858:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.889861:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573429.889866:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573429.889870:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573429.889875:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573429.889882:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.889890:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573429.889895:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573429.889899:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573429.889908:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573429.889913:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573429.889921:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516571776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573429.889951:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573429.889957:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573429.889981:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516571776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573429.890005:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573429.890013:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573429.890018:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573429.890019:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573429.890021:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573429.890033:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573429.890047:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14117 00000400:00000200:0.0:1788573429.890062:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573429.890078:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573429.890086:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573429.890092:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.890101:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573429.890103:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.890105:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573429.890113:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573429.890115:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573429.890122:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573429.890128:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573429.890130:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573429.890132:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573429.890140:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.890149:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573429.890160:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573429.890166:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573429.890171:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573429.890174:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573429.890185:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573429.890188:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573429.890193:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573429.890199:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.890204:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573429.890206:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573429.890211:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.890218:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573429.890229:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573429.890235:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573429.890246:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573429.890255:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573429.890261:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573429.890267:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573429.890273:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573429.890283:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573429.890293:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573429.890302:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516571904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573429.890338:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573429.890346:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573429.890355:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516571904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573429.890377:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573429.890389:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14127) 00000100:00000001:0.0:1788573430.400937:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573430.400953:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573430.400958:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573430.400986:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573430.400997:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573430.401010:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573430.401032:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573430.401036:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573430.401040:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573431.425031:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573431.425037:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573431.425040:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573431.425047:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573431.425053:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573431.425065:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573431.425080:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573431.425083:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573431.425088:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573432.448904:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573432.448911:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573432.448916:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573432.448919:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573432.448939:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573432.448941:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573432.448944:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573432.448946:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573432.448949:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573433.472946:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573433.472953:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573433.472958:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573433.472964:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573433.472970:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573433.472974:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573433.473011:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573433.473016:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573433.473031:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573433.728914:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308793582, at 4308793600 00000100:00000010:0.0:1788573433.728931:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c00. 00000020:00000001:0.0:1788573434.177010:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573434.177014:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573434.177016:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573434.496988:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573434.496993:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573434.496997:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573434.497003:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573434.497007:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573434.497009:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573434.497012:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573434.497014:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573434.497018:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573434.497038:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573434.497041:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573434.497043:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573434.497046:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573434.497050:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573434.497052:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573434.497055:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573434.497057:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573434.497060:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573435.520935:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573435.520942:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573435.520947:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573435.520955:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573435.520962:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573435.520968:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573435.520984:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573435.520987:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573435.520991:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573436.544932:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573436.544938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573436.544943:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573436.544947:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573436.544951:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573436.544957:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573436.544976:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573436.544979:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573436.544982:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573437.568924:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573437.568930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573437.568934:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573437.568939:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573437.568944:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573437.568950:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573437.568985:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573437.568988:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573437.568992:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788573438.081044:0:2386:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788573438.081086:0:2386:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788573438.081093:0:2386:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573438.081096:0:2386:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788573438.081100:0:2386:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573438.081103:0:2386:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788573438.081107:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573438.081110:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793600. 00000400:00000200:0.0:1788573438.081115:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573438.081132:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573438.081137:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcd48 00000400:00000010:0.0:1788573438.081139:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcd48. 00000100:00000001:0.0:1788573438.081143:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573438.081151:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f4000 x1875440516572032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573460 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573438.081173:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573438.081178:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4000 x1875440516572032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573460 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573438.081188:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573438.081191:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573438.081203:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573438.081207:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793300. 00000400:00000200:0.0:1788573438.081212:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573438.081219:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573438.081225:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc908 00000400:00000010:0.0:1788573438.081227:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc908. 00000100:00000001:0.0:1788573438.081232:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573438.081241:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f4e00 x1875440516572160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573460 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573438.081256:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573438.081263:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4e00 x1875440516572160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573460 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573438.081276:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573438.081279:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573438.081288:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573438.081294:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793800. 00000400:00000200:0.0:1788573438.081299:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573438.081303:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573438.081308:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcaa0 00000400:00000010:0.0:1788573438.081313:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcaa0. 00000100:00000001:0.0:1788573438.081328:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573438.081338:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f6a00 x1875440516572288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573460 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573438.081354:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573438.081361:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516572288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573460 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573438.081379:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573438.081384:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573438.081394:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573438.081398:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793a00. 00000400:00000200:0.0:1788573438.081403:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573438.081411:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573438.081418:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc440 00000400:00000010:0.0:1788573438.081421:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc440. 00000100:00000001:0.0:1788573438.081426:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573438.081434:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f7100 x1875440516572416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573460 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573438.081455:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573438.081463:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7100 x1875440516572416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573460 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573438.081476:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573438.081480:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573438.081490:0:2386:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573438.081494:0:2386:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793000. 00000400:00000200:0.0:1788573438.081505:0:2386:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573438.081518:0:2386:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573438.081525:0:2386:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc660 00000400:00000010:0.0:1788573438.081529:0:2386:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc660. 00000100:00000001:0.0:1788573438.081534:0:2386:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573438.081542:0:2386:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f5180 x1875440516572544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573460 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573438.081573:0:2386:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573438.081594:0:2386:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5180 x1875440516572544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573460 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573438.081606:0:2386:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573438.081610:0:2386:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573438.592921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573438.592926:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573438.592931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573438.592937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573438.592944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573438.592947:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573438.592951:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573438.592953:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573438.592956:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573438.848918:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308798701, at 4308798720 00000100:00000010:0.0:1788573438.848937:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7d80. 00000020:00000001:0.0:1788573439.297045:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573439.297050:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573439.297054:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573440.641028:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573440.641037:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573440.641042:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573440.641048:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573440.641055:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573440.641066:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573440.641081:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573440.641094:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573440.641100:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573441.664932:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573441.664938:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573441.664941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573441.664946:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573441.664949:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573441.664951:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573441.664968:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573441.664971:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573441.664975:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573441.664993:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573441.664998:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573441.665000:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573441.665002:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573441.665006:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573441.665011:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573441.665014:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573441.665017:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573441.665021:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573442.689034:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573442.689047:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573442.689057:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573442.689076:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573442.689084:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573442.689097:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573442.689129:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573442.689137:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573442.689151:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573443.713029:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573443.713044:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573443.713046:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573443.713051:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573443.713056:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573443.713060:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573443.713063:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573443.713065:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573443.713069:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573443.968929:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308803822, at 4308803840 00000100:00000010:0.0:1788573443.968947:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7840. 00000020:00000001:0.0:1788573444.417002:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573444.417009:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573444.417013:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573444.736916:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573444.736926:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573444.736948:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573444.736955:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573444.736959:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573444.736962:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573444.737152:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573444.737154:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573444.737157:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573445.760978:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573445.760986:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573445.760991:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573445.760997:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573445.761008:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573445.761018:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573445.761043:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573445.761047:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573445.761050:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573446.784997:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573446.785013:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573446.785026:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573446.785035:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573446.785053:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573446.785067:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573446.785085:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573446.785088:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573446.785092:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573447.808963:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573447.808969:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573447.808972:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573447.808978:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573447.808985:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573447.808990:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573447.809038:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573447.809043:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573447.809052:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573448.832921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573448.832929:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573448.832933:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573448.832943:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573448.832966:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573448.832972:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573448.832979:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573448.832983:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573448.832986:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573449.088935:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308808941, at 4308808960 00000100:00000010:0.0:1788573449.088952:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea71e0. 00000020:00000001:0.0:1788573449.537088:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573449.537093:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573449.537097:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573449.856952:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573449.856960:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573449.856968:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573449.856973:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573449.856978:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573449.856981:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573449.856984:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573449.856987:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573449.856990:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573449.857096:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573449.857102:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573449.857103:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573449.857106:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573449.857109:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573449.857112:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573449.857115:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573449.857116:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573449.857118:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573450.368941:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573450.368954:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573450.368957:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573450.368961:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573450.368967:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573450.368984:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14137 00000400:00000200:0.0:1788573450.368993:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573450.369015:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573450.369018:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573450.369022:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369025:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573450.369028:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369030:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573450.369035:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573450.369037:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573450.369040:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573450.369043:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573450.369046:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573450.369048:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573450.369051:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369053:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573450.369055:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369058:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573450.369061:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573450.369064:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369067:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573450.369070:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573450.369074:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573450.369077:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.369080:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573450.369082:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573450.369090:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573450.369093:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573450.369097:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573450.369099:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.369102:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573450.369105:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573450.369110:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573450.369118:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573450.369131:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573450.369141:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516573952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573450.369152:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573450.369157:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573450.369163:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516573952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573450.369187:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573450.369191:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573450.369195:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573450.369197:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573450.369199:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573450.369203:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573450.369206:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14137 00000400:00000200:0.0:1788573450.369213:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573450.369220:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573450.369224:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573450.369230:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369237:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573450.369242:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369248:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573450.369254:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573450.369258:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573450.369264:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573450.369269:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573450.369274:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573450.369280:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573450.369287:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369294:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573450.369306:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369311:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573450.369316:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573450.369322:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369329:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573450.369335:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573450.369344:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573450.369349:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.369354:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573450.369360:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573450.369367:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.369371:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573450.369377:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573450.369383:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573450.369386:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573450.369388:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.369401:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573450.369405:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573450.369407:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573450.369412:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573450.369418:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573450.369428:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516574080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573450.369444:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573450.369448:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573450.369454:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516574080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573450.369464:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573450.369469:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573450.369475:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573450.369478:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573450.369485:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573450.369489:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573450.369495:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14137 00000400:00000200:0.0:1788573450.369502:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573450.369508:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573450.369511:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573450.369517:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369521:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573450.369524:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369528:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573450.369533:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573450.369534:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573450.369536:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573450.369539:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573450.369543:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573450.369545:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573450.369552:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369554:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573450.369556:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369561:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573450.369563:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573450.369568:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369573:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573450.369575:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573450.369578:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573450.369581:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.369584:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573450.369586:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573450.369590:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.369592:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573450.369602:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573450.369605:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573450.369609:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573450.369612:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.369616:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573450.369618:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573450.369621:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573450.369625:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573450.369628:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573450.369636:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516574208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573450.369647:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573450.369650:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573450.369653:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516574208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573450.369667:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573450.369670:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573450.369674:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573450.369677:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573450.369678:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573450.369680:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573450.369684:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14137 00000400:00000200:0.0:1788573450.369689:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573450.369695:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573450.369700:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573450.369704:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369707:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573450.369711:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369714:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573450.369718:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573450.369726:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573450.369729:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573450.369732:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573450.369736:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573450.369739:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573450.369744:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369748:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573450.369752:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369755:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573450.369759:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573450.369764:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369770:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573450.369773:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573450.369777:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573450.369781:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.369784:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573450.369786:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573450.369791:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.369794:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573450.369797:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573450.369802:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573450.369824:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573450.369825:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.369827:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573450.369828:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573450.369829:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573450.369833:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573450.369840:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573450.369846:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516574336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573450.369859:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573450.369863:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573450.369877:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516574336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573450.369903:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573450.369920:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573450.369927:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573450.369931:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573450.369936:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573450.369943:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573450.369948:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14137 00000400:00000200:0.0:1788573450.369958:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573450.369966:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573450.369972:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573450.369979:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369985:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573450.369991:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.369996:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573450.370001:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573450.370004:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573450.370009:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573450.370014:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573450.370020:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573450.370024:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573450.370031:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.370036:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573450.370041:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573450.370046:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573450.370050:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573450.370054:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573450.370058:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573450.370069:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573450.370074:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573450.370080:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.370084:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573450.370087:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573450.370094:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.370099:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573450.370106:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573450.370111:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573450.370118:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573450.370122:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.370126:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573450.370129:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573450.370133:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573450.370140:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573450.370147:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573450.370156:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516574464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573450.370173:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573450.370179:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573450.370185:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516574464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573450.370204:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573450.370211:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14147) 00000100:00000001:0.0:1788573450.880949:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573450.880955:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573450.880958:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573450.880965:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573450.880973:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573450.880979:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573450.880984:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573450.880988:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573450.880993:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573450.881025:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573450.881051:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573450.881052:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573450.881054:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573450.881060:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573450.881066:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573450.881078:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573450.881082:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573450.881086:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573451.904959:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573451.904966:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573451.904967:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573451.904973:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573451.904977:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573451.904980:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573451.904983:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573451.904986:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573451.904989:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573452.929009:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573452.929020:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573452.929023:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573452.929033:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573452.929041:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573452.929049:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573452.929082:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573452.929086:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573452.929093:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573453.952913:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573453.952921:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573453.952924:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573453.952931:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573453.952938:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573453.952942:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573453.952965:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573453.952968:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573453.952986:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573454.208903:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308814061, at 4308814080 00000100:00000010:0.0:1788573454.208923:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7900. 00000020:00000001:0.0:1788573454.656980:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573454.657029:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573454.657033:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573454.976962:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573454.976968:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573454.976972:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573454.976978:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573454.976982:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573454.976987:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573454.977015:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573454.977020:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573454.977023:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573456.000939:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573456.000945:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573456.000948:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573456.000954:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573456.000961:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573456.000965:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573456.000968:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573456.000970:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573456.000976:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573457.024934:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573457.024941:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573457.024945:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573457.024949:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573457.024956:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573457.024963:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573457.024976:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573457.024980:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573457.024985:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573457.536934:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788573457.536944:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788573457.536955:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788573457.537143:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573457.537162:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573457.537169:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573457.537172:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573458.048900:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573458.048914:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573458.048918:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573458.048923:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573458.048927:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573458.048933:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573458.048946:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573458.048949:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573458.048951:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573459.072908:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573459.072917:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573459.072926:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573459.072934:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573459.072945:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573459.072955:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573459.072962:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573459.072973:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573459.072978:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573459.328911:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308819181, at 4308819200 00000100:00000010:0.0:1788573459.328927:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea74e0. 00000020:00000001:0.0:1788573459.776972:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573459.776976:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573459.776979:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573460.096899:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573460.096908:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573460.096912:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573460.096922:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573460.096930:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573460.096935:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573460.096940:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573460.096943:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573460.096947:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573460.096972:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573460.096976:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573460.096979:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573460.096981:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573460.096983:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573460.096985:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573460.097002:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573460.097004:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573460.097006:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573461.120901:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573461.120907:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573461.120911:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573461.120917:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573461.120923:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573461.120926:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573461.120933:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573461.120936:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573461.120942:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573462.145059:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573462.145063:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573462.145071:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573462.145076:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573462.145080:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573462.145082:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573462.145269:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573462.145272:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573462.145275:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573463.169169:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573463.169172:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573463.169174:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573463.169176:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573463.169178:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573463.169180:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573463.169181:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573463.169183:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573463.169184:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573464.192943:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573464.192952:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573464.192969:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573464.192975:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573464.192984:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573464.192992:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573464.193008:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573464.193012:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573464.193063:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573464.448891:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308824301, at 4308824320 00000100:00000010:0.0:1788573464.448913:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7900. 00000020:00000001:0.0:1788573464.897047:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573464.897051:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573464.897054:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573465.216914:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573465.216920:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573465.216925:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573465.216933:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573465.216939:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573465.216944:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573465.216947:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573465.216951:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573465.216955:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573466.240910:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573466.240919:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573466.240924:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573466.240933:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573466.240940:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573466.240947:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573466.240952:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573466.240957:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573466.240959:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573466.240996:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573466.241001:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573466.241003:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573466.241006:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573466.241012:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573466.241017:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573466.241032:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573466.241034:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573466.241041:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573467.264906:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573467.264910:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573467.264915:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573467.264920:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573467.264943:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573467.264947:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573467.264965:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573467.264968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573467.264970:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573468.288914:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573468.288921:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573468.288925:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573468.288934:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573468.288941:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573468.288945:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573468.288949:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573468.288955:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573468.288962:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573468.288984:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573468.288990:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573468.288995:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573468.289002:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573468.289008:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573468.289012:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573468.289014:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573468.289029:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573468.289032:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573469.312903:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573469.312908:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573469.312912:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573469.312921:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573469.312927:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573469.312931:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573469.312960:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573469.312962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573469.312964:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573469.568908:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308829421, at 4308829440 00000100:00000010:0.0:1788573469.568927:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea71e0. 00000020:00000001:0.0:1788573470.016967:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573470.016973:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573470.016977:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573470.336919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573470.336923:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573470.336927:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573470.336935:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573470.336943:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573470.336949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573470.336962:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573470.336965:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573470.336967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573471.360948:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573471.360955:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573471.360958:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573471.360965:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573471.360978:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573471.360986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573471.361003:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573471.361034:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573471.361038:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573472.384924:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573472.384930:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573472.384935:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573472.384942:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573472.384948:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573472.384954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573472.384968:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573472.384972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573472.384976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573473.408874:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573473.408882:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573473.408885:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573473.408891:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573473.408897:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573473.408901:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573473.408905:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573473.408909:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573473.408915:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573473.408945:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573473.408950:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573473.408971:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573473.408979:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573473.408983:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573473.408988:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573473.408991:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573473.408993:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573473.408995:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573474.432912:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573474.432921:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573474.432926:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573474.432935:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573474.432944:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573474.432948:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573474.432961:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573474.432965:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573474.432968:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573474.688908:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308834541, at 4308834560 00000100:00000010:0.0:1788573474.688927:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7d80. 00000020:00000001:0.0:1788573475.137030:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573475.137033:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573475.137036:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573475.456917:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573475.456926:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573475.456931:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573475.456938:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573475.456945:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573475.456950:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573475.456964:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573475.456968:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573475.456975:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573476.480938:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573476.480947:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573476.480952:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573476.480959:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573476.480965:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573476.480970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573476.480980:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573476.481015:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573476.481020:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573477.504926:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573477.504934:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573477.504936:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573477.504942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573477.504949:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573477.504959:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573477.504967:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573477.504971:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573477.504978:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573477.505040:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573477.505049:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573477.505057:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573477.505066:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573477.505077:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573477.505095:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573477.505110:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573477.505112:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573477.505115:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573478.528911:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573478.528921:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573478.528923:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573478.528931:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573478.528936:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573478.528950:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573478.528955:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573478.528965:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573478.528977:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573478.529037:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573478.529046:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573478.529052:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573478.529058:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573478.529061:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573478.529064:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573478.529065:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573478.529067:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573478.529068:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.552936:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573479.552946:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573479.552950:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573479.552958:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573479.552965:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573479.552971:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.552978:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573479.552983:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.552990:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573479.553037:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573479.553043:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573479.553045:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573479.553054:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573479.553058:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573479.553065:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.553084:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573479.553088:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573479.553092:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573479.808927:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308839661, at 4308839680 00000100:00000010:0.0:1788573479.808951:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7120. 00000020:00000001:0.0:1788573480.257034:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573480.257037:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573480.257040:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573480.576922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573480.576936:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573480.576941:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573480.576950:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573480.576962:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573480.576970:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573480.576980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573480.577003:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573480.577009:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573480.577218:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573480.577220:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573480.577221:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573480.577223:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573480.577225:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573480.577240:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573480.577241:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573480.577242:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573480.577244:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573481.600949:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573481.600958:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573481.600962:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573481.600969:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573481.600976:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573481.600981:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573481.601017:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573481.601019:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573481.601022:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573482.624919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573482.624927:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573482.624930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573482.624937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573482.624943:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573482.624948:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573482.624951:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573482.624956:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573482.624960:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573482.625005:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573482.625009:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573482.625014:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573482.625018:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573482.625022:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573482.625026:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573482.625028:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573482.625032:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573482.625035:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573483.648935:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573483.648945:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573483.648951:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573483.648961:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573483.648977:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573483.648998:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573483.649017:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573483.649044:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573483.649048:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573484.672906:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573484.672916:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573484.672919:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573484.672926:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573484.672931:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573484.672935:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.672941:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573484.672946:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573484.672952:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573484.928881:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308844781, at 4308844800 00000100:00000010:0.0:1788573484.928897:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7000. 00000020:00000001:0.0:1788573485.377002:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573485.377005:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573485.377007:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573485.696917:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573485.696926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573485.696931:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573485.696942:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573485.696951:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573485.696960:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573485.696978:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573485.697006:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573485.697010:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573486.720921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573486.720926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573486.720929:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573486.720933:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573486.720938:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573486.720943:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573486.720957:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573486.720960:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573486.720963:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573488.768907:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573488.768911:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573488.768915:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573488.768940:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573488.768947:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573488.768949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573488.768968:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573488.768971:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573488.768975:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573489.792918:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573489.792926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573489.792929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573489.792938:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573489.792946:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573489.792951:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573489.792958:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573489.792963:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573489.792967:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573489.793046:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573489.793051:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573489.793052:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573489.793054:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573489.793057:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573489.793059:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573489.793060:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573489.793061:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573489.793064:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573490.048900:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308849901, at 4308849920 00000100:00000010:0.0:1788573490.048919:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7360. 00000020:00000001:0.0:1788573490.497040:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573490.497045:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573490.497048:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573490.816897:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573490.816902:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573490.816906:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573490.816911:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573490.816916:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573490.816921:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573490.816933:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573490.816938:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573490.816958:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573491.328928:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573491.328942:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573491.328945:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573491.328949:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573491.328955:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573491.328963:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14178 00000400:00000200:0.0:1788573491.328987:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573491.328995:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573491.328999:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573491.329008:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329012:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573491.329018:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329021:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573491.329025:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573491.329027:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573491.329030:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573491.329033:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573491.329036:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573491.329038:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573491.329041:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329043:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573491.329045:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329047:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573491.329050:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573491.329054:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329056:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573491.329060:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573491.329064:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573491.329067:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.329086:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573491.329089:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573491.329095:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573491.329098:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573491.329102:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573491.329104:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.329108:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573491.329111:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573491.329115:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573491.329122:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573491.329129:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573491.329137:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516579072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573491.329148:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573491.329152:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573491.329157:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516579072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573491.329177:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573491.329181:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573491.329184:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573491.329186:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573491.329187:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573491.329190:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573491.329193:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14178 00000400:00000200:0.0:1788573491.329199:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573491.329203:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573491.329209:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573491.329214:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329220:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573491.329225:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329229:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573491.329257:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573491.329260:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573491.329264:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573491.329269:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573491.329273:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573491.329280:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573491.329289:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329295:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573491.329299:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329305:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573491.329310:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573491.329317:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329322:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573491.329327:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573491.329334:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573491.329340:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.329347:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573491.329353:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573491.329361:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.329366:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573491.329375:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573491.329380:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573491.329388:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573491.329391:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.329395:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573491.329398:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573491.329401:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573491.329409:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573491.329417:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573491.329427:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516579200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573491.329452:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573491.329464:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573491.329475:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516579200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573491.329494:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573491.329500:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573491.329507:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573491.329509:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573491.329512:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573491.329519:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573491.329526:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14178 00000400:00000200:0.0:1788573491.329536:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573491.329544:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573491.329550:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573491.329558:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329564:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573491.329570:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329575:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573491.329580:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573491.329584:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573491.329587:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573491.329591:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573491.329596:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573491.329601:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573491.329610:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329614:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573491.329618:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329623:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573491.329630:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573491.329637:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329653:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573491.329659:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573491.329665:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573491.329673:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.329679:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573491.329684:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573491.329693:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.329697:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573491.329703:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573491.329710:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573491.329716:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573491.329720:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.329726:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573491.329729:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573491.329734:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573491.329742:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573491.329749:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573491.329760:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516579328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573491.329785:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573491.329790:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573491.329799:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516579328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573491.329851:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573491.329859:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573491.329866:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573491.329869:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573491.329873:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573491.329880:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573491.329886:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14178 00000400:00000200:0.0:1788573491.329894:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573491.329911:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573491.329917:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573491.329926:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329932:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573491.329937:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.329945:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573491.329951:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573491.329955:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573491.329958:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573491.329964:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573491.329985:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573491.329992:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573491.329997:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.330004:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573491.330009:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.330013:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573491.330016:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573491.330020:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573491.330024:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573491.330029:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573491.330034:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573491.330040:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.330046:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573491.330049:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573491.330056:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.330060:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573491.330067:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573491.330075:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573491.330081:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573491.330084:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.330088:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573491.330100:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573491.330105:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573491.330114:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573491.330121:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573491.330132:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516579456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573491.330154:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573491.330160:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573491.330170:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516579456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573491.330190:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573491.330194:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573491.330203:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573491.330207:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573491.330210:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573491.330218:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573491.330225:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14178 00000400:00000200:0.0:1788573491.330234:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573491.330244:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573491.330250:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573491.330258:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.330265:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573491.330273:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.330278:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573491.330282:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573491.330285:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573491.330289:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573491.330294:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573491.330299:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573491.330304:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573491.330321:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.330328:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573491.330334:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573491.330341:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573491.330347:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573491.330356:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573491.330364:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573491.330368:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573491.330375:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573491.330383:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.330389:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573491.330394:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573491.330400:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.330403:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573491.330409:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573491.330414:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573491.330420:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573491.330423:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573491.330427:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573491.330429:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573491.330432:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573491.330438:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573491.330444:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573491.330453:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516579584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573491.330466:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573491.330470:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573491.330478:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516579584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573491.330493:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573491.330496:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14188) 00000100:00000001:0.0:1788573492.864914:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573492.864924:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573492.864928:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573492.864935:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573492.864943:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573492.864948:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573492.864965:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573492.864981:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573492.864985:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573493.888902:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573493.888911:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573493.888915:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573493.888924:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573493.888932:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573493.888939:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573493.888955:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573493.888957:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573493.888960:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573494.912951:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573494.912957:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573494.912961:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573494.912983:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573494.912989:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573494.913005:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573494.913011:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573494.913015:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573494.913019:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573495.168879:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308855021, at 4308855040 00000100:00000010:0.0:1788573495.168895:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ba0. 00000020:00000001:0.0:1788573495.617006:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573495.617010:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573495.617012:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573495.936903:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573495.936910:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573495.936915:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573495.936924:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573495.936932:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573495.936948:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573495.936974:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573495.936979:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573495.936986:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788573496.449070:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573496.449086:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573496.449093:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573496.449104:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573496.960918:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573496.960927:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573496.960931:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573496.960938:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573496.960945:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573496.960949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573496.960980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573496.960984:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573496.960987:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573497.984931:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573497.984941:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573497.984947:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573497.984957:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573497.984979:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573497.984983:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573497.984988:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573497.985174:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573497.985178:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573499.008946:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573499.008970:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573499.008973:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573499.008982:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573499.008990:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573499.008996:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573499.009003:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573499.009009:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573499.009014:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573499.009046:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573499.009052:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573499.009056:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573499.009069:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573499.009072:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573499.009074:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573499.009075:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573499.009077:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573499.009080:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573500.032888:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573500.032895:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573500.032899:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573500.032909:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573500.032917:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573500.032923:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573500.032931:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573500.032935:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573500.032938:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573500.288921:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308860141, at 4308860160 00000100:00000010:0.0:1788573500.288938:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674360. 00000020:00000001:0.0:1788573500.737058:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573500.737062:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573500.737065:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573501.056906:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573501.056914:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573501.056916:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573501.056934:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573501.056941:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573501.056946:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573501.056963:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573501.056968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573501.056973:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573501.057010:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573501.057015:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573501.057019:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573501.057028:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573501.057035:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573501.057045:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573501.057065:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573501.057069:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573501.057090:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573502.080922:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573502.080931:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573502.080936:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573502.080944:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573502.080975:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573502.080979:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573502.080995:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573502.080998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573502.081007:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573503.104917:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573503.104925:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573503.104929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573503.104940:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573503.104977:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573503.104983:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573503.105001:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573503.105006:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573503.105012:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573504.128897:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573504.128906:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573504.128910:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573504.128917:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573504.128928:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573504.128933:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573504.128959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573504.128965:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573504.128968:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573505.152966:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573505.152970:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573505.152974:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573505.152981:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573505.152987:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573505.152990:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573505.152994:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573505.152997:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573505.153001:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573505.408917:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308865261, at 4308865280 00000100:00000010:0.0:1788573505.408957:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674720. 00000020:00000001:0.0:1788573505.857193:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573505.857199:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573505.857202:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573506.688964:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573506.688975:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573506.688977:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573506.688980:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573506.688984:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573506.688990:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14194 00000400:00000200:0.0:1788573506.688997:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573506.689018:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573506.689021:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573506.689024:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689028:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573506.689031:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689035:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573506.689040:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573506.689042:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573506.689045:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573506.689049:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573506.689052:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573506.689055:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573506.689060:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689062:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573506.689065:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689068:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573506.689072:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573506.689076:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689079:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573506.689099:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573506.689103:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573506.689105:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.689107:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573506.689109:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573506.689114:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573506.689116:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573506.689120:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573506.689121:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.689124:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573506.689126:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573506.689130:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573506.689137:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573506.689142:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573506.689150:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516580992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573506.689159:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573506.689162:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573506.689166:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516580992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573506.689184:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573506.689187:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573506.689190:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573506.689191:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573506.689192:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573506.689194:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573506.689196:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14194 00000400:00000200:0.0:1788573506.689200:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573506.689202:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573506.689203:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573506.689222:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689223:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573506.689230:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689232:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573506.689234:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573506.689235:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573506.689237:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573506.689239:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573506.689241:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573506.689243:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573506.689245:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689248:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573506.689254:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689260:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573506.689267:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573506.689274:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689280:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573506.689286:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573506.689295:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573506.689303:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.689306:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573506.689328:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573506.689335:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.689340:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573506.689348:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573506.689354:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573506.689357:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573506.689361:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.689366:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573506.689371:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573506.689373:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573506.689379:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573506.689386:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573506.689396:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516581120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573506.689413:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573506.689417:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573506.689424:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516581120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573506.689440:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573506.689443:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573506.689450:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573506.689452:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573506.689457:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573506.689462:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573506.689470:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14194 00000400:00000200:0.0:1788573506.689478:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573506.689487:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573506.689493:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573506.689501:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689505:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573506.689512:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689516:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573506.689519:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573506.689523:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573506.689527:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573506.689530:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573506.689535:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573506.689541:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573506.689547:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689554:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573506.689560:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689566:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573506.689592:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573506.689600:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689605:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573506.689609:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573506.689615:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573506.689620:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.689625:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573506.689627:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573506.689650:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.689655:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573506.689662:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573506.689667:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573506.689674:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573506.689678:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.689684:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573506.689687:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573506.689690:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573506.689697:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573506.689703:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573506.689713:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516581248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573506.689730:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573506.689733:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573506.689739:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516581248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573506.689763:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573506.689768:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573506.689776:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573506.689778:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573506.689781:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573506.689787:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573506.689830:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14194 00000400:00000200:0.0:1788573506.689838:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573506.689843:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573506.689845:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573506.689848:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689850:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573506.689852:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689854:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573506.689857:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573506.689859:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573506.689861:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573506.689863:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573506.689866:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573506.689868:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573506.689871:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689874:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573506.689877:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689884:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573506.689888:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573506.689894:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573506.689900:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573506.689908:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573506.689915:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573506.689920:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.689924:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573506.689928:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573506.689935:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.689940:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573506.689957:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573506.689963:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573506.689970:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573506.689979:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.689986:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573506.689991:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573506.689996:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573506.690005:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573506.690012:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573506.690024:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516581376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573506.690040:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573506.690045:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573506.690054:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516581376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573506.690082:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573506.690087:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573506.690094:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573506.690097:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573506.690101:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573506.690108:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573506.690115:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14194 00000400:00000200:0.0:1788573506.690126:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573506.690134:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573506.690138:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573506.690145:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.690149:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573506.690155:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.690160:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573506.690166:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573506.690170:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573506.690175:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573506.690179:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573506.690184:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573506.690193:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573506.690201:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.690204:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573506.690209:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573506.690214:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573506.690219:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573506.690223:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573506.690228:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573506.690232:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573506.690236:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573506.690241:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.690246:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573506.690250:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573506.690255:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.690260:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573506.690266:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573506.690272:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573506.690277:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573506.690280:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573506.690283:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573506.690285:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573506.690287:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573506.690295:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573506.690302:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573506.690312:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516581504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573506.690325:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573506.690330:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573506.690338:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516581504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573506.690350:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573506.690364:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14204) 00000100:00000001:0.0:1788573507.201017:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573507.201026:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573507.201031:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573507.201039:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573507.201047:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573507.201053:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573507.201058:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573507.201061:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573507.201066:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573507.201095:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573507.201105:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573507.201113:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573507.201133:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573507.201151:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573507.201169:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573507.201196:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573507.201198:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573507.201201:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573508.224920:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573508.224926:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573508.224929:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573508.224933:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573508.224954:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573508.224959:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573508.224970:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573508.224974:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573508.224977:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573509.248908:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573509.248915:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573509.248919:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573509.248924:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573509.248930:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573509.248934:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573509.248979:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573509.248982:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573509.248986:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573510.272906:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573510.272927:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573510.272930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573510.272945:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573510.272949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573510.272954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573510.273969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573510.273972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573510.273974:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573510.528909:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308870381, at 4308870400 00000100:00000010:0.0:1788573510.528930:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916749c0. 00000020:00000001:0.0:1788573510.977109:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573510.977130:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573510.977134:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573511.296930:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573511.296947:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573511.296952:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573511.296958:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573511.296963:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573511.296968:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573511.296993:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573511.296997:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573511.297001:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573512.320902:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573512.320907:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573512.320912:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573512.320921:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573512.320941:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573512.320947:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573512.320951:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573512.320956:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573512.320966:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573513.344989:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573513.344994:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573513.344997:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573513.345001:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573513.345007:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573513.345011:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573513.345022:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573513.345026:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573513.345030:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573514.368944:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573514.368953:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573514.368957:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573514.368967:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573514.368973:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573514.368977:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573514.368981:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573514.368987:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573514.368995:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573515.392904:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573515.392911:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573515.392919:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573515.392939:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573515.392944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573515.392950:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573515.392962:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573515.392965:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573515.392968:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573515.392982:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573515.392986:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573515.392990:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573515.392994:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573515.393000:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573515.393005:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573515.393012:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573515.393015:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573515.393019:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573515.648912:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308875501, at 4308875520 00000100:00000010:0.0:1788573515.648953:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674420. 00000020:00000001:0.0:1788573516.097041:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573516.097045:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573516.097049:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573516.416917:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573516.416936:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573516.416948:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573516.416957:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573516.416965:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573516.416973:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573516.416990:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573516.416993:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573516.416997:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573517.440902:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573517.440911:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573517.440913:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573517.440919:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573517.440939:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573517.440945:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573517.440953:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573517.440974:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573517.440980:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573518.464976:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573518.464981:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573518.464983:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573518.464986:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573518.464989:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573518.464991:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573518.464993:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573518.464994:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573518.464996:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573518.465134:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573518.465137:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573518.465141:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573518.465143:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573518.465145:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573518.465147:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573518.465149:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573518.465150:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573518.465152:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573519.488998:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573519.489003:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573519.489007:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573519.489014:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573519.489036:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573519.489040:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573519.489053:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573519.489058:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573519.489060:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573520.512982:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573520.512993:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573520.512999:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573520.513012:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573520.513026:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573520.513050:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573520.513063:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573520.513069:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573520.513073:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573520.768948:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308880621, at 4308880640 00000100:00000010:0.0:1788573520.768959:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674060. 00000020:00000001:0.0:1788573521.217166:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573521.217170:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573521.217173:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573521.536926:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573521.536937:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573521.536943:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573521.536963:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573521.536971:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573521.536980:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573521.536986:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573521.536989:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573521.536992:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573521.537010:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573521.537015:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573521.537017:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573521.537019:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573521.537022:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573521.537025:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573521.537028:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573521.537032:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573521.537037:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573522.560929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573522.560937:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573522.560941:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573522.560948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573522.560963:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573522.560969:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573522.560972:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573522.560976:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573522.560981:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573522.561012:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573522.561019:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573522.561022:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573522.561027:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573522.561030:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573522.561032:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573522.561034:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573522.561035:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573522.561037:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573523.584964:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573523.584970:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573523.584973:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573523.584980:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573523.584983:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573523.584985:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573523.585008:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573523.585012:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573523.585017:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573524.608929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573524.608939:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573524.608943:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573524.608951:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573524.608957:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573524.608962:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573524.608966:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573524.608971:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573524.608975:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573524.609009:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573524.609013:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573524.609036:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573524.609038:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573524.609041:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573524.609043:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573524.609044:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573524.609046:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573524.609048:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573525.632910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573525.632929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573525.632934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573525.632944:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573525.632953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573525.632959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573525.632979:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573525.632983:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573525.632986:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573525.633005:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573525.633008:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573525.633009:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573525.633012:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573525.633015:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573525.633017:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573525.633020:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573525.633022:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573525.633025:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573525.888898:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308885741, at 4308885760 00000100:00000010:0.0:1788573525.888914:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674000. 00000020:00000001:0.0:1788573526.337085:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573526.337089:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573526.337098:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573526.657057:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573526.657074:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573526.657082:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573526.657088:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573526.657092:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573526.657098:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573526.657101:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573526.657116:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573526.657120:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573527.168905:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573527.168917:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573527.168934:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573527.168938:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573527.168944:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573527.168952:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14214 00000400:00000200:0.0:1788573527.168963:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573527.168971:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573527.168975:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573527.168979:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.168983:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573527.168987:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.168990:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573527.168994:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573527.168996:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573527.169000:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573527.169003:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573527.169006:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573527.169008:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573527.169011:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169013:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573527.169014:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169016:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573527.169018:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573527.169021:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169023:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573527.169025:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573527.169027:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573527.169029:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169047:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573527.169049:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573527.169055:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573527.169058:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573527.169061:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573527.169063:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169065:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573527.169067:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573527.169079:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573527.169087:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573527.169094:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573527.169105:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516583552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573527.169116:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573527.169121:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573527.169126:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516583552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573527.169147:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573527.169150:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573527.169154:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573527.169156:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573527.169157:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573527.169160:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573527.169164:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14214 00000400:00000200:0.0:1788573527.169169:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573527.169172:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573527.169175:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573527.169178:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169181:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573527.169183:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169186:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573527.169194:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573527.169195:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573527.169197:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573527.169200:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573527.169202:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573527.169204:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573527.169208:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169210:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573527.169213:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169215:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573527.169217:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573527.169219:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169222:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573527.169224:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573527.169230:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573527.169232:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169234:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573527.169238:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573527.169241:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169243:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573527.169246:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573527.169249:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573527.169251:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573527.169253:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169254:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573527.169256:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573527.169258:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573527.169262:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573527.169266:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573527.169271:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516583680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573527.169279:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573527.169286:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573527.169291:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516583680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573527.169300:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573527.169302:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573527.169306:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573527.169308:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573527.169309:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573527.169312:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573527.169314:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14214 00000400:00000200:0.0:1788573527.169366:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573527.169369:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573527.169375:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573527.169378:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169380:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573527.169382:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169384:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573527.169387:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573527.169388:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573527.169390:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573527.169392:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573527.169394:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573527.169397:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573527.169400:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169402:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573527.169404:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169405:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573527.169408:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573527.169410:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169417:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573527.169419:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573527.169422:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573527.169424:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169427:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573527.169429:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573527.169432:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169434:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573527.169438:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573527.169440:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573527.169443:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573527.169444:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169446:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573527.169448:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573527.169450:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573527.169454:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573527.169458:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573527.169463:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516583808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573527.169471:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573527.169473:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573527.169477:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516583808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573527.169485:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573527.169488:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573527.169491:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573527.169493:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573527.169494:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573527.169497:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573527.169499:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14214 00000400:00000200:0.0:1788573527.169503:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573527.169511:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573527.169514:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573527.169517:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169519:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573527.169521:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169523:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573527.169526:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573527.169527:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573527.169529:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573527.169531:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573527.169533:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573527.169536:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573527.169539:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169541:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573527.169544:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169546:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573527.169548:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573527.169551:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169554:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573527.169556:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573527.169559:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573527.169561:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169564:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573527.169566:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573527.169569:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169571:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573527.169574:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573527.169576:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573527.169579:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573527.169581:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169582:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573527.169590:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573527.169592:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573527.169596:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573527.169599:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573527.169605:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516583936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573527.169614:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573527.169616:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573527.169620:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516583936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573527.169628:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573527.169631:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573527.169634:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573527.169636:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573527.169638:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573527.169640:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573527.169643:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14214 00000400:00000200:0.0:1788573527.169647:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573527.169650:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573527.169652:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573527.169655:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169657:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573527.169660:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169662:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573527.169664:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573527.169666:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573527.169668:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573527.169669:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573527.169672:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573527.169674:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573527.169681:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169683:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573527.169685:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169687:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573527.169690:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573527.169692:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573527.169695:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573527.169697:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573527.169699:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573527.169702:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169704:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573527.169706:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573527.169708:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169710:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573527.169713:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573527.169715:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573527.169718:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573527.169719:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573527.169721:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573527.169722:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573527.169724:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573527.169728:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573527.169731:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573527.169736:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516584064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573527.169744:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573527.169746:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573527.169750:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516584064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573527.169759:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573527.169763:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14224) 00000100:00000001:0.0:1788573527.680907:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573527.680915:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573527.680945:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573527.680951:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573527.680955:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573527.680959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573527.680976:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573527.680980:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573527.680993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573528.704972:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573528.704979:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573528.704984:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573528.704993:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573528.705001:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573528.705012:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573528.705019:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573528.705022:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573528.705026:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573529.728904:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573529.728910:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573529.728915:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573529.728936:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573529.728942:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573529.728946:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573529.728967:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573529.728971:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573529.728974:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573530.752987:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573530.752993:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573530.752995:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573530.753001:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573530.753008:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573530.753013:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573530.753019:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573530.753023:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573530.753030:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573531.008915:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308890861, at 4308890880 00000100:00000010:0.0:1788573531.008961:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674240. 00000020:00000001:0.0:1788573531.457069:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573531.457073:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573531.457076:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573531.776987:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573531.776993:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573531.776997:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573531.777004:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573531.777007:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573531.777010:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573531.777278:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573531.777282:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573531.777286:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573532.800889:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573532.800898:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573532.800902:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573532.800910:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573532.800917:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573532.800923:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573532.800938:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573532.800943:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573532.800950:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573533.824917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573533.824926:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573533.824943:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573533.824951:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573533.824957:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573533.824961:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573533.824965:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573533.824970:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573533.824974:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573533.825001:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573533.825006:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573533.825009:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573533.825012:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573533.825017:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573533.825020:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573533.825021:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573533.825022:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573533.825041:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573534.848971:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573534.848976:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573534.848980:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573534.848984:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573534.848990:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573534.848993:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573534.848996:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573534.848999:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573534.849001:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788573535.361195:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573535.361211:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573535.361217:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573535.361219:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573535.872906:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573535.872915:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573535.872920:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573535.872928:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573535.872942:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573535.872947:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573535.872954:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573535.872957:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573535.872963:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573535.872993:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573535.872997:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573535.872998:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573535.873000:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573535.873003:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573535.873005:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573535.873007:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573535.873009:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573535.873011:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573536.128904:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308895982, at 4308896000 00000100:00000010:0.0:1788573536.128924:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7420. 00000020:00000001:0.0:1788573536.577034:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573536.577037:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573536.577040:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573536.896923:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573536.896930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573536.896946:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573536.896952:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573536.896956:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573536.896959:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573536.896977:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573536.896982:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573536.896987:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573537.920986:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573537.920992:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573537.920997:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573537.921006:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573537.921013:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573537.921017:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573537.921022:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573537.921027:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573537.921032:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573537.921050:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573537.921053:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573537.921054:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573537.921056:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573537.921058:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573537.921060:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573537.921073:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573537.921076:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573537.921079:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573538.944886:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573538.944894:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573538.944896:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573538.944901:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573538.944916:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573538.944918:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573538.944920:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573538.944922:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573538.944929:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573539.968928:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573539.968933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573539.968966:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573539.968973:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573539.968978:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573539.968983:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573539.968992:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573539.968997:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573539.969002:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573540.992917:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573540.992926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573540.992931:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573540.992954:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573540.992962:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573540.992968:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573540.993012:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573540.993017:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573540.993025:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573541.248924:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308901101, at 4308901120 00000100:00000010:0.0:1788573541.248953:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c60. 00000020:00000001:0.0:1788573541.696975:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573541.696978:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573541.696981:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573542.016994:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573542.017000:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573542.017004:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573542.017009:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573542.017013:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573542.017015:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573542.017033:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573542.017037:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573542.017040:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573543.040905:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573543.040913:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573543.040918:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573543.040926:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573543.040933:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573543.040954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573543.041179:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573543.041200:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573543.041203:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573544.064923:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573544.064929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573544.064934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573544.064940:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573544.064963:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573544.064969:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573544.064977:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573544.064986:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573544.064990:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573544.065091:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573544.065097:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573544.065101:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573544.065104:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573544.065107:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573544.065109:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573544.065218:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573544.065222:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573544.065225:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573545.088906:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573545.088913:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573545.088916:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573545.088926:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573545.088935:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573545.088942:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573545.088960:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573545.088964:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573545.088968:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573545.088999:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573545.089002:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573545.089004:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573545.089006:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573545.089010:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573545.089013:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573545.089016:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573545.089018:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573545.089021:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573546.112930:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573546.112937:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573546.112941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573546.112959:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573546.112966:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573546.112971:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573546.112991:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573546.112995:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573546.112999:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573546.368959:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308906221, at 4308906240 00000100:00000010:0.0:1788573546.368977:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7a80. 00000020:00000001:0.0:1788573546.817054:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573546.817058:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573546.817062:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573547.136918:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573547.136925:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573547.136931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573547.136938:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573547.136944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573547.136962:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573547.137007:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573547.137010:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573547.137013:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573548.160929:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573548.160935:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573548.160940:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573548.160960:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573548.160967:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573548.160973:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573548.160999:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573548.161002:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573548.161006:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573549.184986:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573549.184991:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573549.184995:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573549.185001:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573549.185007:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573549.185017:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573549.185020:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573549.185022:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573549.185024:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573550.208980:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573550.208986:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573550.208989:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573550.208995:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573550.209001:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573550.209005:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573550.209009:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573550.209011:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573550.209014:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573550.209132:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573550.209136:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573550.209139:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573550.209144:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573550.209150:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573550.209153:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573550.209157:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573550.209160:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573550.209163:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573550.209190:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573550.209194:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573550.209196:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573550.209198:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573550.209201:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573550.209203:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573550.209205:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573550.209206:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573550.209209:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573551.232926:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573551.232933:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573551.232937:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573551.232943:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573551.232948:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573551.232970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573551.233020:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573551.233039:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573551.233042:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573551.488907:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308911341, at 4308911360 00000100:00000010:0.0:1788573551.488925:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea79c0. 00000020:00000001:0.0:1788573551.936926:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573551.936931:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573551.936935:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573552.256931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573552.256936:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573552.256944:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573552.256970:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573552.256976:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573552.256982:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573552.257020:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573552.257023:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573552.257027:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573553.280921:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573553.280926:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573553.280930:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573553.280935:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573553.280941:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573553.280948:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573553.280973:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573553.280976:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573553.280979:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573554.304953:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573554.304972:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573554.304978:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573554.304984:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573554.304992:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573554.304998:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573554.305014:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573554.305020:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573554.305028:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573555.328947:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573555.328953:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573555.328971:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573555.328984:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573555.328990:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573555.328993:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573555.329001:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573555.329005:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573555.329008:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573556.352919:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573556.352927:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573556.352931:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573556.352937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573556.352944:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573556.352950:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573556.352957:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573556.352972:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573556.352975:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573556.608928:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308916461, at 4308916480 00000100:00000010:0.0:1788573556.608937:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f60. 00000020:00000001:0.0:1788573557.056954:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573557.056958:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573557.056982:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573557.376955:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573557.376971:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573557.376975:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573557.376981:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573557.376986:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573557.376991:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573557.377000:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573557.377003:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573557.377006:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573558.400900:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573558.400905:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573558.400909:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573558.400915:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573558.400922:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573558.400925:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573558.400939:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573558.400941:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573558.400955:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573559.425008:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573559.425021:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573559.425037:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573559.425052:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573559.425064:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573559.425077:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573559.425098:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573559.425103:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573559.425108:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573560.448904:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573560.448911:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573560.448915:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573560.448923:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573560.448929:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573560.448935:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573560.448949:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573560.448954:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573560.448958:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573561.472919:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573561.472928:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573561.472934:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573561.472951:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573561.472956:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573561.472960:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573561.472982:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573561.472986:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573561.472992:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573561.728933:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308921581, at 4308921600 00000100:00000010:0.0:1788573561.728953:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7540. 00000020:00000001:0.0:1788573562.177018:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573562.177022:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573562.177025:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573563.008984:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573563.008997:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573563.009000:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573563.009019:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573563.009024:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573563.009031:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14250 00000400:00000200:0.0:1788573563.009040:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573563.009046:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573563.009049:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573563.009053:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009056:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573563.009059:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009062:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573563.009066:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573563.009068:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573563.009070:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573563.009073:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573563.009076:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573563.009079:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573563.009082:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009084:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573563.009086:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009088:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573563.009091:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573563.009094:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009097:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573563.009100:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573563.009104:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573563.009107:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.009110:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573563.009112:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573563.009119:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573563.009122:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573563.009126:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573563.009133:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.009136:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573563.009138:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573563.009143:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573563.009150:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573563.009158:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573563.009167:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516588032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573563.009179:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573563.009183:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573563.009188:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516588032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573563.009213:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573563.009216:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573563.009220:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573563.009221:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573563.009223:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573563.009226:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573563.009229:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14250 00000400:00000200:0.0:1788573563.009235:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573563.009239:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573563.009242:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573563.009245:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009248:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573563.009252:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009258:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573563.009263:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573563.009267:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573563.009270:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573563.009273:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573563.009279:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573563.009288:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573563.009304:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009314:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573563.009324:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009331:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573563.009340:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573563.009351:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009357:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573563.009366:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573563.009374:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573563.009378:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.009380:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573563.009383:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573563.009386:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.009399:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573563.009407:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573563.009413:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573563.009425:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573563.009426:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.009432:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573563.009437:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573563.009442:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573563.009450:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573563.009455:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573563.009465:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516588160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573563.009503:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573563.009513:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573563.009526:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516588160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573563.009550:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573563.009566:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573563.009576:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573563.009583:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573563.009589:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573563.009604:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573563.009616:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14250 00000400:00000200:0.0:1788573563.009644:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573563.009653:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573563.009660:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573563.009671:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009682:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573563.009690:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009703:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573563.009709:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573563.009712:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573563.009718:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573563.009724:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573563.009731:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573563.009737:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573563.009743:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009751:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573563.009759:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009768:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573563.009776:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573563.009789:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573563.009794:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573563.009803:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573563.009833:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573563.009837:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.009840:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573563.009850:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573563.009855:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.009859:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573563.009862:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573563.009865:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573563.009868:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573563.009874:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.009877:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573563.009878:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573563.009881:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573563.009886:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573563.009891:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573563.009901:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516588288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573563.009912:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573563.009914:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573563.009926:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516588288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573563.009948:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573563.009954:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573563.009963:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573563.009980:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573563.009984:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573563.009988:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573563.009993:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14250 00000400:00000200:0.0:1788573563.010009:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573563.010019:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573563.010021:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573563.010025:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.010034:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573563.010037:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.010052:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573563.010059:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573563.010063:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573563.010067:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573563.010070:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573563.010072:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573563.010078:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573563.010085:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.010093:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573563.010100:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.010103:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573563.010106:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573563.010117:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573563.010124:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573563.010134:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573563.010141:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573563.010154:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.010172:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573563.010184:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573563.010195:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.010199:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573563.010212:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573563.010226:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573563.010237:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573563.010241:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.010257:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573563.010266:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573563.010271:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573563.010282:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573563.010288:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573563.010308:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516588416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573563.010360:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573563.010367:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573563.010378:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516588416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573563.010403:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573563.010407:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573563.010415:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573563.010420:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573563.010425:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573563.010434:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573563.010436:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14250 00000400:00000200:0.0:1788573563.010445:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573563.010451:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573563.010460:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573563.010467:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.010476:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573563.010481:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.010484:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573563.010488:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573563.010490:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573563.010494:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573563.010501:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573563.010505:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573563.010508:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573563.010517:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.010525:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573563.010533:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573563.010540:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573563.010554:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573563.010563:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573563.010572:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573563.010574:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573563.010577:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573563.010583:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.010600:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573563.010614:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573563.010623:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.010626:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573563.010632:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573563.010641:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573563.010651:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573563.010657:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.010662:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573563.010666:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573563.010677:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573563.010689:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573563.010702:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573563.010723:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516588544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573563.010740:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573563.010747:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573563.010762:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516588544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573563.010774:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573563.010787:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14260) 00000100:00000001:0.0:1788573563.520985:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573563.520995:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573563.521008:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573563.521022:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573563.521033:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573563.521058:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573563.521061:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573563.521063:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573563.521065:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573563.521099:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573563.521104:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573563.521105:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573563.521108:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573563.521112:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573563.521116:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573563.521118:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573563.521119:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573563.521122:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573564.544915:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573564.544925:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573564.544931:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573564.544942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573564.544951:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573564.544957:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573564.545023:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573564.545027:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573564.545031:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573564.545051:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573564.545056:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573564.545058:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573564.545062:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573564.545065:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573564.545069:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573564.545072:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573564.545075:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573564.545078:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573566.592970:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573566.592994:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573566.592998:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573566.593003:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573566.593007:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573566.593011:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573566.593015:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573566.593027:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573566.593032:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573566.848916:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308926702, at 4308926720 00000100:00000010:0.0:1788573566.848936:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea73c0. 00000020:00000001:0.0:1788573567.297021:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573567.297025:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573567.297028:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573567.617109:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573567.617129:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573567.617132:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573567.617139:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573567.617170:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573567.617174:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573567.617177:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573567.617179:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573567.617197:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573568.128930:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573568.128941:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573568.128944:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573568.128947:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573568.128953:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573568.128962:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14255 00000400:00000200:0.0:1788573568.128971:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573568.128995:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573568.128998:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573568.129003:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129006:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573568.129010:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129012:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573568.129016:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573568.129017:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573568.129020:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573568.129023:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573568.129042:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573568.129046:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573568.129050:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129052:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573568.129054:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129056:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573568.129059:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573568.129062:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129065:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573568.129068:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573568.129072:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573568.129075:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.129077:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573568.129080:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573568.129087:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573568.129091:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573568.129095:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573568.129097:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.129100:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573568.129103:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573568.129107:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573568.129113:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573568.129118:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573568.129126:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516588672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573568.129134:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573568.129137:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573568.129140:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516588672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573568.129159:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573568.129162:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573568.129171:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573568.129173:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573568.129174:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573568.129177:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573568.129181:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14255 00000400:00000200:0.0:1788573568.129185:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573568.129188:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573568.129191:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573568.129204:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129214:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573568.129235:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129241:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573568.129249:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573568.129254:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573568.129262:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573568.129270:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573568.129281:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573568.129287:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573568.129294:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129297:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573568.129303:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129306:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573568.129311:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573568.129317:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129324:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573568.129329:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573568.129341:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573568.129345:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.129349:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573568.129354:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573568.129363:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.129365:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573568.129367:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573568.129373:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573568.129377:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573568.129378:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.129382:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573568.129384:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573568.129386:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573568.129390:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573568.129398:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573568.129415:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516588800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573568.129435:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573568.129438:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573568.129450:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516588800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573568.129458:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573568.129462:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573568.129464:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573568.129465:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573568.129466:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573568.129470:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573568.129476:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14255 00000400:00000200:0.0:1788573568.129483:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573568.129491:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573568.129493:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573568.129498:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129506:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573568.129510:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129517:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573568.129518:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573568.129522:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573568.129527:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573568.129534:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573568.129543:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573568.129544:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573568.129547:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129553:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573568.129556:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129561:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573568.129563:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573568.129572:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129578:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573568.129580:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573568.129584:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573568.129587:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.129589:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573568.129593:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573568.129598:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.129599:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573568.129602:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573568.129609:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573568.129617:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573568.129618:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.129620:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573568.129621:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573568.129622:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573568.129625:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573568.129630:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573568.129635:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516588928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573568.129655:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573568.129658:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573568.129664:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516588928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573568.129690:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573568.129696:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573568.129702:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573568.129703:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573568.129707:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573568.129712:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573568.129724:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14255 00000400:00000200:0.0:1788573568.129736:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573568.129739:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573568.129742:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573568.129745:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129747:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573568.129752:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129757:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573568.129765:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573568.129768:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573568.129773:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573568.129778:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573568.129785:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573568.129792:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573568.129799:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129803:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573568.129804:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129805:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573568.129847:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573568.129856:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573568.129861:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573568.129872:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573568.129876:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573568.129881:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.129887:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573568.129891:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573568.129894:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.129895:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573568.129897:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573568.129899:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573568.129901:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573568.129902:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.129903:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573568.129904:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573568.129905:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573568.129909:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573568.129917:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573568.129925:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516589056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573568.129939:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573568.129944:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573568.129951:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516589056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573568.129964:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573568.129966:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573568.129969:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573568.129985:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573568.129989:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573568.129993:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573568.130005:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14255 00000400:00000200:0.0:1788573568.130017:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573568.130024:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573568.130034:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573568.130038:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.130048:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573568.130052:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.130056:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573568.130064:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573568.130066:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573568.130074:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573568.130078:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573568.130083:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573568.130086:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573568.130091:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.130100:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573568.130104:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573568.130110:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573568.130119:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573568.130128:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573568.130133:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573568.130137:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573568.130142:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573568.130152:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.130162:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573568.130167:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573568.130174:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.130182:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573568.130187:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573568.130191:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573568.130198:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573568.130207:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573568.130210:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573568.130214:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573568.130217:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573568.130224:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573568.130232:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573568.130243:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516589184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573568.130265:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573568.130272:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573568.130280:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516589184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573568.130295:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573568.130301:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14265) 00000100:00000001:0.0:1788573568.640906:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573568.640918:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573568.640924:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573568.640941:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573568.640948:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573568.640955:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573568.640989:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573568.640993:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573568.640996:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573569.664921:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573569.664936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573569.664941:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573569.664950:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573569.664956:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573569.664963:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573569.664973:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573569.665008:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573569.665023:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573570.688949:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573570.688959:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573570.688966:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573570.688976:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573570.689028:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573570.689036:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573570.689052:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573570.689056:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573570.689059:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788573571.201233:0:2389:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788573571.201317:0:2389:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788573571.201331:0:2389:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573571.201336:0:2389:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788573571.201342:0:2389:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573571.201348:0:2389:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788573571.201354:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573571.201359:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793700. 00000400:00000200:0.0:1788573571.201367:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573571.201374:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573571.201384:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fca18 00000400:00000010:0.0:1788573571.201388:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fca18. 00000100:00000001:0.0:1788573571.201396:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573571.201407:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c1f80 x1875440516588672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573571.201433:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573571.201438:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c1f80 x1875440516588672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573593 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573571.201449:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573571.201453:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573571.201470:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573571.201475:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793000. 00000400:00000200:0.0:1788573571.201483:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573571.201492:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573571.201498:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fce58 00000400:00000010:0.0:1788573571.201502:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fce58. 00000100:00000001:0.0:1788573571.201505:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573571.201534:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c2300 x1875440516588800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573571.201549:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573571.201570:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c2300 x1875440516588800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573593 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573571.201590:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573571.201595:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573571.201611:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573571.201620:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793a00. 00000400:00000200:0.0:1788573571.201629:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573571.201645:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573571.201657:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc000 00000400:00000010:0.0:1788573571.201663:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc000. 00000100:00000001:0.0:1788573571.201673:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573571.201683:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c0700 x1875440516588928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573571.201704:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573571.201716:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c0700 x1875440516588928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573593 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573571.201765:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573571.201769:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573571.201776:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573571.201782:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793800. 00000400:00000200:0.0:1788573571.201788:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573571.201802:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573571.201869:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcee0 00000400:00000010:0.0:1788573571.201874:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcee0. 00000100:00000001:0.0:1788573571.201886:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573571.201896:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c1c00 x1875440516589056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573571.201932:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573571.201950:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c1c00 x1875440516589056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573593 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573571.201991:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573571.201996:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573571.202013:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573571.202021:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793300. 00000400:00000200:0.0:1788573571.202027:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573571.202043:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573571.202053:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc198 00000400:00000010:0.0:1788573571.202061:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc198. 00000100:00000001:0.0:1788573571.202068:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573571.202078:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c2680 x1875440516589184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573571.202124:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573571.202133:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c2680 x1875440516589184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573593 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573571.202156:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573571.202162:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573571.712975:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573571.712993:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573571.712997:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573571.713005:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573571.713011:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573571.713016:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573571.713043:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573571.713048:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573571.713052:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573571.968912:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308931822, at 4308931840 00000100:00000010:0.0:1788573571.968928:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea75a0. 00000020:00000001:0.0:1788573572.417034:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573572.417038:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573572.417042:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573572.736922:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573572.736941:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573572.736944:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573572.736950:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573572.736956:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573572.736961:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573572.737415:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573572.737418:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573572.737421:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573573.248938:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788573573.248953:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788573573.248959:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:0.0:1788573573.760902:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573573.760910:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573573.760915:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573573.760925:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573573.760931:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573573.760936:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573573.760942:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573573.760945:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573573.760951:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573573.761013:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573573.761019:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573573.761022:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573573.761025:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573573.761028:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573573.761029:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573573.761031:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573573.761032:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573573.761033:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573574.785081:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573574.785085:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573574.785087:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573574.785089:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573574.785091:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573574.785093:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573574.786166:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573574.786169:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573574.786188:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573575.808935:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573575.808946:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573575.808948:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573575.808954:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573575.808963:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573575.808969:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573575.808974:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573575.808978:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573575.808980:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573576.832911:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573576.832919:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573576.832924:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573576.832935:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573576.832942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573576.832947:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573576.832962:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573576.832967:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573576.832973:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573577.088905:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308936943, at 4308936960 00000100:00000010:0.0:1788573577.088931:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7420. 00000020:00000001:0.0:1788573577.537045:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573577.537049:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573577.537052:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573577.856936:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573577.856940:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573577.856946:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573577.856952:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573577.856958:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573577.856962:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573577.856967:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573577.856970:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573577.856973:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573578.368939:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573578.368950:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573578.368952:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573578.368974:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573578.368979:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573578.369005:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14265 00000400:00000200:0.0:1788573578.369015:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573578.369022:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573578.369025:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573578.369028:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369031:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573578.369033:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369036:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573578.369040:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573578.369042:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573578.369045:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573578.369048:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573578.369050:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573578.369053:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573578.369056:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369058:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573578.369060:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369062:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573578.369064:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573578.369068:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369070:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573578.369073:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573578.369077:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573578.369080:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.369082:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573578.369084:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573578.369093:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573578.369096:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573578.369109:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573578.369119:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.369122:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573578.369125:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573578.369130:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573578.369138:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573578.369144:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573578.369162:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516589952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573578.369174:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573578.369178:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573578.369183:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516589952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573578.369203:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573578.369206:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573578.369210:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573578.369212:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573578.369214:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573578.369217:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573578.369220:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14265 00000400:00000200:0.0:1788573578.369225:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573578.369230:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573578.369235:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573578.369238:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369241:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573578.369246:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369249:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573578.369253:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573578.369257:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573578.369261:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573578.369265:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573578.369271:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573578.369287:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573578.369296:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369302:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573578.369307:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369314:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573578.369318:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573578.369325:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369331:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573578.369333:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573578.369341:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573578.369347:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.369353:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573578.369358:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573578.369366:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.369371:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573578.369378:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573578.369383:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573578.369390:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573578.369393:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.369396:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573578.369400:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573578.369402:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573578.369408:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573578.369413:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573578.369422:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516590080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573578.369437:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573578.369442:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573578.369447:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516590080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573578.369463:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573578.369476:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573578.369481:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573578.369483:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573578.369485:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573578.369492:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573578.369499:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14265 00000400:00000200:0.0:1788573578.369507:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573578.369515:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573578.369519:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573578.369525:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369529:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573578.369534:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369537:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573578.369542:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573578.369545:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573578.369548:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573578.369553:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573578.369557:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573578.369561:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573578.369568:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369571:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573578.369577:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369583:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573578.369587:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573578.369593:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369597:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573578.369601:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573578.369604:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573578.369609:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.369613:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573578.369622:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573578.369627:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.369632:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573578.369638:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573578.369643:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573578.369649:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573578.369652:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.369657:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573578.369660:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573578.369664:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573578.369672:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573578.369677:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573578.369687:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516590208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573578.369702:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573578.369705:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573578.369711:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516590208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573578.369727:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573578.369731:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573578.369740:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573578.369744:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573578.369756:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573578.369766:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573578.369773:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14265 00000400:00000200:0.0:1788573578.369784:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573578.369792:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573578.369795:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573578.369801:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369806:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573578.369826:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369833:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573578.369835:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573578.369836:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573578.369839:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573578.369841:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573578.369845:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573578.369849:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573578.369856:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369860:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573578.369865:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369869:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573578.369872:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573578.369877:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573578.369882:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573578.369887:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573578.369891:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573578.369894:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.369899:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573578.369901:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573578.369906:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.369910:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573578.369916:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573578.369920:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573578.369926:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573578.369927:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.369928:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573578.369929:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573578.369931:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573578.369934:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573578.369940:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573578.369948:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516590336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573578.369976:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573578.369980:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573578.369987:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516590336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573578.370009:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573578.370013:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573578.370017:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573578.370020:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573578.370023:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573578.370028:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573578.370033:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14265 00000400:00000200:0.0:1788573578.370038:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573578.370043:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573578.370045:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573578.370051:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.370053:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573578.370056:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.370059:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573578.370063:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573578.370065:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573578.370067:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573578.370071:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573578.370075:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573578.370077:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573578.370082:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.370084:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573578.370086:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573578.370089:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573578.370097:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573578.370102:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573578.370107:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573578.370110:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573578.370114:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573578.370116:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.370118:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573578.370119:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573578.370121:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.370125:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573578.370132:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573578.370135:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573578.370138:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573578.370141:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573578.370144:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573578.370145:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573578.370148:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573578.370153:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573578.370158:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573578.370166:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516590464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573578.370173:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573578.370174:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573578.370177:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516590464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573578.370186:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573578.370191:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14275) 00000100:00000001:0.0:1788573578.880954:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573578.880962:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573578.880966:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573578.880975:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573578.880983:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573578.881017:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573578.881020:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573578.881022:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573578.881025:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573579.904914:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573579.904924:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573579.904929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573579.904945:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573579.904950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573579.904955:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573579.904976:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573579.904980:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573579.904984:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573580.928947:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573580.928953:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573580.928958:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573580.928965:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573580.928972:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573580.928978:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573580.929002:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573580.929007:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573580.929015:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573581.952919:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573581.952927:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573581.952932:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573581.952948:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573581.952955:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573581.952960:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573581.952974:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573581.952977:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573581.952981:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573582.208944:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308942061, at 4308942080 00000100:00000010:0.0:1788573582.208962:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7b40. 00000020:00000001:0.0:1788573582.656974:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573582.656978:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573582.656982:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573582.976930:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573582.976955:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573582.976960:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573582.976966:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573582.976972:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573582.976977:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573582.976987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573582.976991:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573582.977008:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573583.488934:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573583.488948:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573583.488951:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573583.488955:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573583.488961:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573583.488969:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14270 00000400:00000200:0.0:1788573583.488978:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573583.488984:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573583.488988:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573583.488992:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489010:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573583.489013:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489017:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573583.489021:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573583.489023:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573583.489026:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573583.489030:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573583.489032:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573583.489035:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573583.489039:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489041:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573583.489043:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489045:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573583.489048:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573583.489068:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489071:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573583.489075:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573583.489078:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573583.489081:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.489084:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573583.489086:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573583.489092:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573583.489096:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573583.489100:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573583.489103:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.489106:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573583.489109:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573583.489113:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573583.489121:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573583.489127:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573583.489137:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516590592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573583.489148:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573583.489152:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573583.489158:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516590592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573583.489182:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573583.489187:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573583.489191:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573583.489193:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573583.489195:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573583.489198:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573583.489200:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14270 00000400:00000200:0.0:1788573583.489205:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573583.489209:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573583.489212:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573583.489221:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489225:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573583.489229:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489233:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573583.489238:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573583.489242:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573583.489244:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573583.489249:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573583.489253:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573583.489259:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573583.489268:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489273:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573583.489278:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489284:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573583.489287:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573583.489292:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489300:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573583.489306:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573583.489315:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573583.489320:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.489325:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573583.489332:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573583.489337:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.489342:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573583.489347:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573583.489353:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573583.489358:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573583.489361:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.489367:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573583.489369:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573583.489372:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573583.489380:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573583.489392:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573583.489403:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516590720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573583.489418:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573583.489423:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573583.489432:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516590720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573583.489450:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573583.489456:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573583.489461:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573583.489465:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573583.489467:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573583.489470:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573583.489476:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14270 00000400:00000200:0.0:1788573583.489485:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573583.489492:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573583.489495:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573583.489502:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489507:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573583.489512:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489516:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573583.489521:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573583.489524:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573583.489528:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573583.489532:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573583.489538:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573583.489542:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573583.489553:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489560:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573583.489573:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489577:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573583.489584:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573583.489591:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489596:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573583.489599:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573583.489602:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573583.489604:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.489607:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573583.489611:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573583.489616:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.489621:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573583.489627:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573583.489633:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573583.489639:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573583.489642:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.489648:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573583.489651:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573583.489655:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573583.489662:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573583.489668:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573583.489678:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516590848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573583.489695:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573583.489700:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573583.489709:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516590848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573583.489733:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573583.489738:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573583.489744:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573583.489748:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573583.489757:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573583.489765:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573583.489771:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14270 00000400:00000200:0.0:1788573583.489781:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573583.489788:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573583.489793:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573583.489797:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489799:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573583.489802:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489806:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573583.489831:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573583.489832:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573583.489835:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573583.489837:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573583.489840:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573583.489843:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573583.489847:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489850:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573583.489854:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489859:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573583.489862:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573583.489869:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573583.489873:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573583.489877:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573583.489882:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573583.489888:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.489892:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573583.489896:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573583.489902:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.489905:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573583.489919:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573583.489924:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573583.489929:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573583.489933:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.489938:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573583.489942:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573583.489947:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573583.489955:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573583.489961:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573583.489972:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516590976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573583.490015:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573583.490020:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573583.490030:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516590976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573583.490052:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573583.490058:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573583.490064:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573583.490066:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573583.490070:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573583.490077:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573583.490084:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14270 00000400:00000200:0.0:1788573583.490094:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573583.490098:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573583.490104:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573583.490111:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.490117:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573583.490122:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.490126:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573583.490131:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573583.490142:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573583.490146:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573583.490149:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573583.490154:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573583.490160:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573583.490168:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.490176:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573583.490179:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573583.490186:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573583.490191:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573583.490197:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573583.490203:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573583.490208:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573583.490213:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573583.490219:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.490223:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573583.490226:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573583.490231:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.490236:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573583.490243:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573583.490248:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573583.490254:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573583.490257:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573583.490260:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573583.490264:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573583.490269:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573583.490276:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573583.490283:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573583.490292:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516591104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573583.490307:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573583.490313:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573583.490330:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516591104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573583.490359:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573583.490366:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14281) 00000100:00000001:0.0:1788573584.000913:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573584.000923:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573584.000928:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573584.000936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573584.000944:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573584.000951:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573584.000983:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573584.000987:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573584.000990:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573586.048923:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573586.048929:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573586.048933:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573586.048939:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573586.048945:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573586.048949:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573586.048956:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573586.048961:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573586.048964:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573587.072973:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573587.072986:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573587.073002:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573587.073011:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573587.073023:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573587.073028:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573587.073036:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573587.073042:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573587.073046:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573587.328948:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308947181, at 4308947200 00000100:00000010:0.0:1788573587.328966:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7600. 00000020:00000001:0.0:1788573587.777051:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573587.777056:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573587.777061:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573588.097039:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573588.097045:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573588.097049:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573588.097056:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573588.097060:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573588.097063:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573588.097116:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573588.097117:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573588.097119:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573588.608874:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573588.608886:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573588.608888:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573588.608892:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573588.608897:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573588.608904:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14276 00000400:00000200:0.0:1788573588.608912:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573588.608918:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573588.608921:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573588.608923:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.608926:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573588.608928:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.608930:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573588.608934:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573588.608936:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573588.608938:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573588.608940:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573588.608941:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573588.608943:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573588.608945:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.608947:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573588.608948:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.608950:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573588.608986:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573588.608989:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573588.608991:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573588.608993:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573588.608995:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573588.608997:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.608999:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573588.609001:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573588.609006:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573588.609009:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573588.609013:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573588.609014:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609016:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573588.609018:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573588.609022:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573588.609030:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573588.609035:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573588.609042:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516591232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573588.609050:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573588.609054:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573588.609058:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516591232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573588.609075:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573588.609078:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573588.609081:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573588.609082:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573588.609083:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573588.609085:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573588.609087:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14276 00000400:00000200:0.0:1788573588.609092:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573588.609101:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573588.609104:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573588.609109:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609112:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573588.609116:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609118:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573588.609123:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573588.609125:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573588.609128:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573588.609131:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573588.609134:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573588.609138:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573588.609143:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609147:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573588.609151:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609153:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573588.609156:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573588.609162:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609166:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573588.609168:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573588.609174:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573588.609179:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609181:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573588.609184:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573588.609188:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609191:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573588.609194:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573588.609199:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573588.609203:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573588.609204:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609207:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573588.609215:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573588.609217:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573588.609222:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573588.609228:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573588.609238:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516591360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573588.609251:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573588.609255:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573588.609264:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516591360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573588.609287:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573588.609291:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573588.609296:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573588.609298:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573588.609299:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573588.609300:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573588.609305:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14276 00000400:00000200:0.0:1788573588.609311:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573588.609318:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573588.609320:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573588.609326:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609330:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573588.609334:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609338:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573588.609343:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573588.609347:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573588.609350:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573588.609353:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573588.609357:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573588.609362:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573588.609396:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609403:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573588.609410:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609414:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573588.609418:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573588.609424:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609427:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573588.609429:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573588.609434:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573588.609439:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609444:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573588.609446:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573588.609450:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609454:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573588.609459:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573588.609465:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573588.609472:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573588.609475:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609482:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573588.609486:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573588.609491:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573588.609500:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573588.609507:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573588.609517:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516591488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573588.609527:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573588.609529:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573588.609536:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516591488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573588.609547:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573588.609549:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573588.609561:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573588.609566:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573588.609568:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573588.609572:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573588.609578:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14276 00000400:00000200:0.0:1788573588.609584:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573588.609591:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573588.609595:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573588.609599:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609604:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573588.609608:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609611:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573588.609616:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573588.609618:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573588.609621:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573588.609624:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573588.609628:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573588.609630:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573588.609634:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609637:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573588.609640:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609644:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573588.609648:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573588.609653:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609659:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573588.609660:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573588.609663:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573588.609666:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609670:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573588.609673:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573588.609679:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609683:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573588.609686:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573588.609690:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573588.609692:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573588.609696:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609702:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573588.609703:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573588.609705:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573588.609708:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573588.609713:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573588.609721:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516591616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573588.609733:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573588.609737:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573588.609752:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516591616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573588.609759:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573588.609761:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573588.609764:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573588.609766:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573588.609770:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573588.609775:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573588.609779:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14276 00000400:00000200:0.0:1788573588.609783:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573588.609788:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573588.609793:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573588.609797:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609800:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573588.609804:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609828:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573588.609830:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573588.609831:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573588.609832:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573588.609834:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573588.609835:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573588.609837:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573588.609842:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609846:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573588.609848:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609852:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573588.609855:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573588.609860:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573588.609865:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573588.609869:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573588.609873:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573588.609877:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609880:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573588.609882:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573588.609886:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609888:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573588.609892:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573588.609895:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573588.609899:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573588.609902:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573588.609905:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573588.609907:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573588.609910:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573588.609914:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573588.609919:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573588.609926:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516591744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573588.609949:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573588.609952:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573588.609957:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516591744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573588.609978:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573588.609984:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14286) 00000100:00000001:0.0:1788573589.120988:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573589.120992:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573589.120996:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573589.121001:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573589.121005:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573589.121008:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573589.121013:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573589.121015:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573589.121017:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573590.144913:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573590.144921:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573590.144924:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573590.144930:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573590.144935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573590.144940:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573590.144954:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573590.144959:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573590.144966:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573590.145146:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573590.145148:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573590.145148:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573590.145150:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573590.145152:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573590.145153:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573590.145155:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573590.145156:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573590.145157:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573591.168967:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573591.168975:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573591.168998:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573591.169007:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573591.169018:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573591.169024:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573591.169036:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573591.169039:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573591.169044:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573592.193132:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573592.193141:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573592.193148:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573592.193157:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573592.193166:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573592.193170:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573592.193173:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573592.193175:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573592.193178:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573592.448923:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308952301, at 4308952320 00000100:00000010:0.0:1788573592.448934:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ea0. 00000020:00000001:0.0:1788573592.896997:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573592.897000:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573592.897003:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573593.216961:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573593.216978:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573593.216982:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573593.216989:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573593.216994:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573593.216999:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573593.217014:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573593.217017:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573593.217020:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573593.729082:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573593.729095:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573593.729097:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573593.729101:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573593.729106:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573593.729113:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14281 00000400:00000200:0.0:1788573593.729139:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573593.729144:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573593.729147:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573593.729150:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729152:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573593.729154:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729156:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573593.729159:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573593.729161:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573593.729163:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573593.729165:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573593.729167:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573593.729168:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573593.729171:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729172:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573593.729173:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729175:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573593.729177:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573593.729179:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729181:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573593.729183:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573593.729185:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573593.729187:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.729189:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573593.729190:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573593.729196:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573593.729198:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573593.729201:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573593.729202:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.729204:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573593.729206:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573593.729211:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573593.729223:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573593.729228:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573593.729235:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516591872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573593.729244:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573593.729247:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573593.729250:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516591872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573593.729267:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573593.729270:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573593.729273:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573593.729274:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573593.729275:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573593.729277:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573593.729279:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14281 00000400:00000200:0.0:1788573593.729282:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573593.729284:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573593.729286:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573593.729288:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729291:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573593.729292:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729294:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573593.729304:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573593.729308:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573593.729310:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573593.729312:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573593.729315:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573593.729317:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573593.729323:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729331:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573593.729339:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729342:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573593.729347:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573593.729353:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729357:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573593.729361:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573593.729367:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573593.729370:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.729373:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573593.729376:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573593.729380:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.729385:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573593.729392:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573593.729395:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573593.729402:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573593.729405:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.729410:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573593.729414:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573593.729417:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573593.729426:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573593.729432:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573593.729442:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516592000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573593.729458:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573593.729463:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573593.729474:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516592000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573593.729491:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573593.729497:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573593.729503:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573593.729513:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573593.729516:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573593.729518:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573593.729522:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14281 00000400:00000200:0.0:1788573593.729528:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573593.729536:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573593.729539:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573593.729543:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729546:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573593.729551:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729554:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573593.729558:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573593.729560:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573593.729563:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573593.729566:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573593.729570:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573593.729574:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573593.729579:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729583:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573593.729587:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729590:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573593.729594:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573593.729597:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729599:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573593.729602:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573593.729606:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573593.729610:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.729615:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573593.729617:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573593.729621:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.729627:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573593.729631:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573593.729634:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573593.729639:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573593.729642:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.729646:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573593.729648:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573593.729650:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573593.729657:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573593.729663:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573593.729671:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516592128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573593.729686:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573593.729688:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573593.729693:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516592128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573593.729706:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573593.729710:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573593.729716:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573593.729719:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573593.729721:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573593.729724:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573593.729728:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14281 00000400:00000200:0.0:1788573593.729731:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573593.729733:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573593.729734:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573593.729736:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729740:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573593.729743:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729748:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573593.729752:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573593.729758:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573593.729760:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573593.729762:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573593.729765:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573593.729768:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573593.729773:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729777:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573593.729781:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729786:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573593.729790:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573593.729797:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573593.729801:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573593.729803:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573593.729806:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573593.729838:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.729844:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573593.729847:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573593.729853:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.729859:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573593.729865:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573593.729870:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573593.729874:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573593.729876:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.729879:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573593.729880:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573593.729882:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573593.729889:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573593.729906:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573593.729913:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516592256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573593.729923:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573593.729926:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573593.729943:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516592256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573593.729954:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573593.729960:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573593.729963:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573593.729966:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573593.729968:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573593.729973:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573593.729978:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14281 00000400:00000200:0.0:1788573593.729983:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573593.729990:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573593.729994:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573593.730000:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.730004:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573593.730007:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.730013:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573593.730017:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573593.730019:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573593.730022:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573593.730026:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573593.730030:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573593.730035:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573593.730039:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.730042:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573593.730045:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573593.730047:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573593.730052:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573593.730057:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573593.730061:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573593.730067:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573593.730072:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573593.730076:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.730078:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573593.730082:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573593.730086:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.730090:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573593.730094:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573593.730097:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573593.730099:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573593.730099:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573593.730101:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573593.730103:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573593.730106:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573593.730112:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573593.730118:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573593.730125:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516592384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573593.730140:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573593.730144:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573593.730149:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516592384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573593.730161:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573593.730167:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14291) 00000100:00000001:0.0:1788573594.241049:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573594.241054:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573594.241056:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573594.241060:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573594.241067:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573594.241071:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573594.241080:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573594.241083:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573594.241089:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573595.264998:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573595.265016:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573595.265027:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573595.265043:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573595.265055:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573595.265061:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573595.265075:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573595.265080:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573595.265086:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573596.288922:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573596.288932:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573596.288937:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573596.288945:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573596.288951:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573596.288957:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573596.288962:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573596.288966:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573596.288970:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573597.312928:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573597.312933:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573597.312937:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573597.312940:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573597.312946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573597.312950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573597.312953:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573597.312957:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573597.312961:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573597.313060:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573597.313063:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573597.313064:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573597.313067:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573597.313069:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573597.313074:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573597.313888:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573597.313891:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573597.313893:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573597.568924:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308957422, at 4308957440 00000100:00000010:0.0:1788573597.568942:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7120. 00000020:00000001:0.0:1788573598.016990:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573598.017013:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573598.017016:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573598.336933:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573598.336942:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573598.336949:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573598.336958:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573598.336975:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573598.336981:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573598.336995:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573598.336999:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573598.337003:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573598.848910:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573598.848923:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573598.848927:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573598.848932:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573598.848939:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573598.848950:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14286 00000400:00000200:0.0:1788573598.848960:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573598.848968:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573598.848972:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573598.848975:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.848979:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573598.848982:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.848985:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573598.848990:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573598.848992:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573598.848994:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573598.848998:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573598.849001:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573598.849004:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573598.849008:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849010:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573598.849031:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849033:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573598.849037:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573598.849041:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849044:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573598.849048:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573598.849051:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573598.849054:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.849057:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573598.849059:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573598.849066:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573598.849070:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573598.849075:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573598.849077:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.849080:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573598.849084:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573598.849089:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573598.849097:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573598.849105:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573598.849114:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516592512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573598.849127:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573598.849131:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573598.849136:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516592512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573598.849161:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573598.849165:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573598.849169:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573598.849171:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573598.849174:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573598.849177:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573598.849190:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14286 00000400:00000200:0.0:1788573598.849200:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573598.849210:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573598.849215:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573598.849222:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849228:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573598.849238:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849244:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573598.849250:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573598.849254:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573598.849259:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573598.849262:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573598.849269:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573598.849276:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573598.849285:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849289:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573598.849293:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849296:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573598.849301:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573598.849306:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849309:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573598.849311:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573598.849318:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573598.849324:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.849328:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573598.849333:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573598.849341:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.849345:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573598.849354:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573598.849359:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573598.849367:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573598.849378:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.849382:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573598.849386:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573598.849391:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573598.849400:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573598.849408:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573598.849421:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516592640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573598.849435:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573598.849441:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573598.849450:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516592640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573598.849469:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573598.849476:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573598.849485:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573598.849490:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573598.849494:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573598.849497:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573598.849500:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14286 00000400:00000200:0.0:1788573598.849508:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573598.849517:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573598.849521:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573598.849527:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849530:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573598.849536:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849541:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573598.849548:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573598.849551:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573598.849554:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573598.849559:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573598.849564:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573598.849576:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573598.849585:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849591:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573598.849595:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849598:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573598.849601:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573598.849608:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849614:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573598.849619:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573598.849625:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573598.849631:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.849635:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573598.849638:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573598.849642:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.849647:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573598.849653:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573598.849658:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573598.849661:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573598.849663:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.849665:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573598.849667:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573598.849669:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573598.849677:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573598.849684:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573598.849693:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516592768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573598.849707:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573598.849713:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573598.849722:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516592768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573598.849747:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573598.849758:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573598.849766:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573598.849770:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573598.849774:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573598.849794:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573598.849803:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14286 00000400:00000200:0.0:1788573598.849843:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573598.849848:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573598.849851:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573598.849854:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849858:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573598.849861:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849866:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573598.849873:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573598.849877:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573598.849882:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573598.849887:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573598.849893:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573598.849898:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573598.849907:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849914:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573598.849919:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849924:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573598.849929:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573598.849937:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573598.849944:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573598.849947:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573598.849950:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573598.849957:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.849962:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573598.849977:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573598.849985:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.849990:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573598.849998:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573598.850003:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573598.850009:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573598.850014:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.850018:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573598.850021:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573598.850025:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573598.850033:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573598.850040:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573598.850053:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516592896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573598.850070:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573598.850075:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573598.850084:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516592896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573598.850102:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573598.850108:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573598.850119:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573598.850123:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573598.850126:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573598.850129:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573598.850136:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14286 00000400:00000200:0.0:1788573598.850145:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573598.850153:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573598.850158:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573598.850161:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.850168:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573598.850171:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.850186:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573598.850192:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573598.850196:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573598.850201:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573598.850204:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573598.850210:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573598.850217:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573598.850225:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.850232:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573598.850239:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573598.850246:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573598.850253:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573598.850260:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573598.850266:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573598.850270:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573598.850276:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573598.850281:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.850285:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573598.850289:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573598.850295:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.850301:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573598.850307:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573598.850313:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573598.850320:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573598.850324:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573598.850326:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573598.850330:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573598.850334:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573598.850341:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573598.850349:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573598.850362:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516593024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573598.850394:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573598.850399:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573598.850409:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516593024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573598.850435:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573598.850443:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14296) 00000100:00000001:0.0:1788573599.360906:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573599.360912:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573599.360917:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573599.360927:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573599.360935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573599.360942:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573599.360957:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573599.360962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573599.360967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573600.384922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573600.384931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573600.384936:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573600.384942:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573600.384956:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573600.384961:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573600.384975:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573600.384978:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573600.384981:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573600.384997:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573600.385002:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573600.385004:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573600.385007:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573600.385009:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573600.385012:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573600.385014:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573600.385015:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573600.385017:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573601.408912:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573601.408917:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573601.408936:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573601.408942:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573601.408945:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573601.408948:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573601.408971:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573601.408974:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573601.408977:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573602.432922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573602.432933:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573602.432938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573602.432946:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573602.432954:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573602.432960:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573602.432989:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573602.432992:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573602.432994:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573602.688920:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308962541, at 4308962560 00000100:00000010:0.0:1788573602.688939:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7d80. 00000020:00000001:0.0:1788573603.137002:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573603.137005:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573603.137008:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573603.456948:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573603.456952:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573603.456956:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573603.456962:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573603.456968:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573603.456973:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573603.456993:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573603.456996:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573603.456999:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573604.480928:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573604.480939:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573604.480945:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573604.480954:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573604.480962:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573604.480970:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573604.480987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573604.480998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573604.481003:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573605.504925:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573605.504936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573605.504942:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573605.504950:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573605.504956:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573605.504962:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573605.504966:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573605.504971:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573605.504977:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573606.528968:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573606.528975:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573606.528980:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573606.528986:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573606.528992:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573606.528998:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573606.529014:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573606.529018:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573606.529023:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573607.552928:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573607.552934:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573607.552939:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573607.552947:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573607.552955:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573607.552962:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573607.553005:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573607.553008:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573607.553011:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573607.808934:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308967661, at 4308967680 00000100:00000010:0.0:1788573607.808955:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7de0. 00000020:00000001:0.0:1788573608.257041:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573608.257046:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573608.257050:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573608.576936:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573608.576945:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573608.576950:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573608.576978:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573608.576984:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573608.576988:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573608.577007:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573608.577011:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573608.577014:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573609.600857:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573609.600864:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573609.600866:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573609.600872:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573609.600878:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573609.600882:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573609.600885:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573609.600888:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573609.600892:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573609.600935:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573609.600940:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573609.600943:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573609.600950:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573609.600955:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573609.600958:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573609.600961:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573609.600963:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573609.600965:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573610.624919:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573610.624924:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573610.624928:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573610.624934:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573610.624940:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573610.624943:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573610.624950:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573610.624956:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573610.624959:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573611.648907:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573611.648914:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573611.648917:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573611.648926:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573611.648933:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573611.648954:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573611.648964:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573611.648967:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573611.648971:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573612.160934:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788573612.160945:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788573612.160951:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788573612.161130:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573612.161149:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573612.161156:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573612.161159:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573612.672916:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573612.672921:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573612.672925:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573612.672930:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573612.672933:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573612.672935:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573612.672965:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573612.672968:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573612.672970:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573612.928930:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308972782, at 4308972800 00000100:00000010:0.0:1788573612.928951:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea71e0. 00000020:00000001:0.0:1788573613.377073:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573613.377078:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573613.377082:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573613.696967:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573613.696976:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573613.696980:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573613.696994:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573613.697001:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573613.697009:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573613.697023:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573613.697029:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573613.697033:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573613.697079:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573613.697082:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573613.697110:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573613.697114:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573613.697117:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573613.697119:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573613.697122:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573613.697124:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573613.697126:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573614.720909:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573614.720916:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573614.720920:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573614.720927:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573614.720934:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573614.720941:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573614.720950:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573614.720955:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573614.720960:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573615.744924:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573615.744933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573615.744939:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573615.744955:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573615.744967:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573615.744973:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573615.744999:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573615.745004:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573615.745010:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573616.768895:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573616.768904:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573616.768909:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573616.768916:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573616.768922:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573616.768928:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573616.768934:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573616.768937:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573616.768942:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573616.768959:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573616.768964:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573616.768967:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573616.768973:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573616.768996:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573616.768999:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573616.769001:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573616.769002:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573616.769005:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573617.792909:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573617.792918:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573617.792923:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573617.792931:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573617.792939:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573617.792946:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573617.792967:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573617.792970:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573617.792973:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573618.048915:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308977901, at 4308977920 00000100:00000010:0.0:1788573618.048932:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea78a0. 00000020:00000001:0.0:1788573618.497006:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573618.497011:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573618.497014:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573618.816936:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573618.816948:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573618.816958:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573618.816966:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573618.816973:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573618.816982:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573618.816989:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573618.816999:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573618.817011:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573619.328960:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573619.328974:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573619.328976:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573619.328982:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573619.328990:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573619.329001:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14306 00000400:00000200:0.0:1788573619.329015:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573619.329040:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573619.329044:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573619.329048:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329052:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573619.329055:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329058:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573619.329062:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573619.329064:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573619.329067:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573619.329070:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573619.329073:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573619.329075:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573619.329079:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329081:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573619.329083:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329086:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573619.329089:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573619.329092:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329095:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573619.329098:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573619.329113:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573619.329117:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.329120:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573619.329122:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573619.329131:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573619.329135:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573619.329139:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573619.329141:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.329144:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573619.329147:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573619.329151:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573619.329160:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573619.329174:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573619.329184:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516595072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573619.329196:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573619.329209:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573619.329214:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516595072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573619.329241:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573619.329245:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573619.329249:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573619.329250:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573619.329252:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573619.329255:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573619.329258:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14306 00000400:00000200:0.0:1788573619.329263:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573619.329268:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573619.329271:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573619.329274:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329278:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573619.329280:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329282:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573619.329284:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573619.329286:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573619.329288:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573619.329292:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573619.329296:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573619.329301:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573619.329309:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329313:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573619.329326:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329333:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573619.329339:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573619.329347:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329357:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573619.329362:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573619.329370:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573619.329375:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.329380:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573619.329386:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573619.329390:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.329392:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573619.329400:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573619.329407:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573619.329411:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573619.329413:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.329417:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573619.329420:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573619.329425:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573619.329433:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573619.329440:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573619.329451:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516595200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573619.329470:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573619.329475:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573619.329483:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516595200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573619.329502:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573619.329507:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573619.329515:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573619.329520:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573619.329545:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573619.329552:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573619.329560:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14306 00000400:00000200:0.0:1788573619.329583:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573619.329593:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573619.329597:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573619.329606:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329612:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573619.329620:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329627:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573619.329633:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573619.329637:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573619.329641:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573619.329647:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573619.329654:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573619.329662:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573619.329668:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329671:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573619.329679:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329685:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573619.329690:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573619.329696:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329699:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573619.329703:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573619.329709:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573619.329716:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.329721:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573619.329725:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573619.329731:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.329735:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573619.329750:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573619.329757:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573619.329762:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573619.329766:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.329771:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573619.329775:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573619.329780:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573619.329789:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573619.329798:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573619.329832:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516595328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573619.329842:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573619.329847:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573619.329857:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516595328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573619.329880:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573619.329885:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573619.329895:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573619.329899:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573619.329903:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573619.329910:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573619.329917:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14306 00000400:00000200:0.0:1788573619.329925:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573619.329932:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573619.329937:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573619.329943:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329948:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573619.329955:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.329961:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573619.329967:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573619.329978:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573619.329982:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573619.329988:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573619.329995:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573619.330002:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573619.330011:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.330016:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573619.330023:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.330028:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573619.330032:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573619.330040:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573619.330046:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573619.330051:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573619.330057:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573619.330064:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.330069:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573619.330074:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573619.330080:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.330085:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573619.330093:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573619.330098:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573619.330101:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573619.330104:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.330107:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573619.330111:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573619.330115:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573619.330122:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573619.330129:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573619.330138:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516595456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573619.330152:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573619.330157:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573619.330174:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516595456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573619.330196:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573619.330203:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573619.330209:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573619.330213:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573619.330216:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573619.330222:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573619.330228:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14306 00000400:00000200:0.0:1788573619.330237:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573619.330243:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573619.330249:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573619.330256:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.330259:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573619.330265:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.330271:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573619.330276:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573619.330279:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573619.330284:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573619.330289:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573619.330293:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573619.330296:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573619.330301:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.330304:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573619.330310:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573619.330315:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573619.330322:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573619.330329:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573619.330335:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573619.330346:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573619.330353:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573619.330358:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.330362:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573619.330366:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573619.330373:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.330378:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573619.330385:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573619.330390:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573619.330396:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573619.330400:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.330407:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573619.330411:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573619.330416:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573619.330424:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573619.330431:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573619.330444:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516595584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573619.330461:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573619.330467:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573619.330475:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516595584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573619.330501:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573619.330509:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14316) 00000100:00000001:0.0:1788573619.840957:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573619.840967:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573619.840974:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573619.840982:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573619.840988:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573619.840996:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573619.841011:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573619.841020:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573619.841033:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573619.841085:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573619.841106:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573619.841108:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573619.841111:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573619.841114:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573619.841116:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573619.841119:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573619.841121:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573619.841123:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573620.864969:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573620.864980:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573620.864988:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573620.864999:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573620.865009:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573620.865019:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573620.865032:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573620.865035:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573620.865040:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573622.912964:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573622.912971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573622.912977:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573622.912985:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573622.912994:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573622.913001:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573622.913047:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573622.913052:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573622.913060:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573623.168978:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308983021, at 4308983040 00000100:00000010:0.0:1788573623.168993:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea74e0. 00000020:00000001:0.0:1788573623.617042:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573623.617046:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573623.617048:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573623.936931:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573623.936938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573623.936943:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573623.936948:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573623.936953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573623.936960:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573623.937018:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573623.937021:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573623.937024:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573624.961014:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573624.961024:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573624.961034:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573624.961054:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573624.961063:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573624.961070:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573624.961080:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573624.961083:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573624.961086:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573625.984910:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573625.984917:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573625.984920:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573625.984925:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573625.984930:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573625.984934:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573625.984946:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573625.984950:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573625.984955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573627.008896:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573627.008906:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573627.008909:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573627.008915:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573627.008918:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573627.008923:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573627.008930:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573627.008940:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573627.008951:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573627.009023:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573627.009028:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573627.009030:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573627.009032:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573627.009034:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573627.009036:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573627.009062:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573627.009066:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573627.009077:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573628.032949:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573628.032956:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573628.032959:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573628.032964:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573628.032971:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573628.032977:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573628.032981:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573628.032985:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573628.032989:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573628.288920:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308988142, at 4308988160 00000100:00000010:0.0:1788573628.288940:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7720. 00000020:00000001:0.0:1788573628.736983:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573628.736986:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573628.736989:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573629.056915:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573629.056923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573629.056926:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573629.056932:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573629.056954:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573629.056959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.056981:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573629.056984:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.056987:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.569213:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573629.569220:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573629.569226:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573629.569236:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4000 x1875440516596352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.569252:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573629.569255:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573629.569261:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573629.569266:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573629.569272:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573629.569278:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516596352:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573629.569295:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573629.569301:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573629.569305:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573629.569308:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573629.569312:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573629.569316:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573629.569324:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573629.569330:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573629.569338:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573629.569346:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516596352, portal 25 00000100:00000001:0.0:1788573629.569349:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573629.569352:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605320704 : -123400104230912 : ffff8fc4ac9f4000) 00000100:00000040:0.0:1788573629.569358:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4000 x1875440516596352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.569371:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573629.569376:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573629.569404:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788573629.569408:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516596352, offset 0 00000400:00000200:0.0:1788573629.569417:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573629.569428:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573629.569439:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573629.569450:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573629.569460:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573629.569468:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573629.569476:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573629.569482:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546311:546311:256:4294967295] 192.168.202.128@tcp LPNI seq info [546234:546234:8:4294967295] 00000400:00000200:0.0:1788573629.569489:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573629.569505:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573629.569515:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573629.569520:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788573629.569531:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573629.569542:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1788573629.569562:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573629.569567:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788573629.569575:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573629.569582:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573629.569589:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788573629.569594:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788573629.569602:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573629.569609:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4000 x1875440516596352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.569626:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573629.569633:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4000 x1875440516596352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.569646:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.569650:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573629.569655:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573629.569665:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573629.569669:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4308993261 00000100:00000010:0.0:1788573629.569677:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573629.569685:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573629.569693:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4308993261 00000100:00000010:0.0:1788573629.569698:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50360. 00000100:00000001:0.0:1788573629.569703:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573629.569707:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573629.569709:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.569712:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573629.569717:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573629/real 1788573629] req@ffff8fc4ac9f4000 x1875440516596352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573629.569729:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573629.569732:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573629.569742:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573629.569745:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573629.569749:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573629.569756:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573629.569765:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4000 x1875440516596352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573629.569788:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4000 x1875440516596352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.569805:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573629.569827:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.569829:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573629.569831:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573629.569835:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4000 x1875440516596352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.569846:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573629.569855:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4000 x1875440516596352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.569865:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573629.569870:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.569875:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573629.569878:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.569884:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573629.569887:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573629.569891:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573629.569900:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573629.569907:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573629.569918:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4000 x1875440516596352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573629.569941:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516596352:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573629.569965:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.569971:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573629.569979:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4000 x1875440516596352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570002:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573629.570006:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573629.570011:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573629.570017:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573629.570020:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573629.570023:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573629.570028:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573629.570032:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35400. 02000000:00000001:0.0:1788573629.570036:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573629.570040:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573629.570042:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4000. 00000100:00000001:0.0:1788573629.570045:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573629.570048:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.570050:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.570057:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573629.570060:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.570065:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573629.570068:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573629.570071:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573629.570077:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5180 x1875440516596480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570086:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573629.570089:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573629.570093:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573629.570096:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573629.570101:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573629.570106:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516596480:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573629.570113:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573629.570117:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573629.570119:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573629.570134:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573629.570139:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573629.570144:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573629.570153:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573629.570157:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573629.570163:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573629.570169:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516596480, portal 10 00000100:00000001:0.0:1788573629.570173:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573629.570176:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605325184 : -123400104226432 : ffff8fc4ac9f5180) 00000100:00000040:0.0:1788573629.570186:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5180 x1875440516596480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570200:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573629.570205:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573629.570209:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788573629.570212:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516596480, offset 0 00000400:00000200:0.0:1788573629.570219:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573629.570226:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573629.570233:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573629.570240:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573629.570245:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573629.570251:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573629.570258:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573629.570263:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546312:546312:256:4294967295] 192.168.202.128@tcp LPNI seq info [546235:546235:8:4294967295] 00000400:00000200:0.0:1788573629.570269:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573629.570280:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573629.570289:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573629.570293:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788573629.570298:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573629.570302:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1788573629.570319:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573629.570323:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788573629.570334:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573629.570342:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573629.570349:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788573629.570351:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788573629.570355:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573629.570362:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5180 x1875440516596480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570401:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573629.570408:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5180 x1875440516596480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570442:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.570447:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573629.570455:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573629.570460:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573629.570465:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573629.570468:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308989459 00000100:00000001:0.0:1788573629.570471:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308993261 00000100:00000001:0.0:1788573629.570473:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308989459 00000100:00000001:0.0:1788573629.570475:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308993261 00000100:00000001:0.0:1788573629.570477:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573629.570478:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.570480:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573629.570483:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573629/real 1788573629] req@ffff8fc4ac9f5180 x1875440516596480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573629.570489:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573629.570490:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573629.570491:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573629.570493:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573629.570494:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573629.570496:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573629.570503:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516596480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573629.570509:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516596480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570513:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573629.570514:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.570515:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573629.570516:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573629.570518:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516596480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570523:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573629.570526:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516596480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570530:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573629.570531:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.570532:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573629.570533:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.570534:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573629.570536:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573629.570537:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573629.570539:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573629.570541:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573629.570543:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516596480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573629.570548:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516596480:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573629.570550:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573629.570552:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7100 x1875440516596608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570575:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573629.570576:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573629.570580:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573629.570582:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573629.570583:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573629.570585:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516596608:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573629.570588:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573629.570589:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573629.570590:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573629.570591:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573629.570594:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573629.570595:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573629.570598:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573629.570600:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573629.570602:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573629.570604:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516596608, portal 10 00000100:00000001:0.0:1788573629.570606:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573629.570607:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605333248 : -123400104218368 : ffff8fc4ac9f7100) 00000100:00000040:0.0:1788573629.570611:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7100 x1875440516596608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570615:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573629.570617:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573629.570618:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788573629.570620:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516596608, offset 0 00000400:00000200:0.0:1788573629.570622:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573629.570625:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573629.570628:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573629.570630:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573629.570632:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573629.570634:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573629.570635:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573629.570641:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546313:546313:256:4294967295] 192.168.202.128@tcp LPNI seq info [546236:546236:8:4294967295] 00000400:00000200:0.0:1788573629.570644:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573629.570649:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573629.570652:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573629.570654:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788573629.570657:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573629.570659:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1788573629.570666:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573629.570667:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788573629.570669:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573629.570672:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573629.570674:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788573629.570675:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788573629.570677:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573629.570679:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7100 x1875440516596608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570685:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573629.570687:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7100 x1875440516596608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570691:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.570692:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573629.570694:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573629.570696:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573629.570698:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573629.570700:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308989466 00000100:00000001:0.0:1788573629.570701:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308993261 00000100:00000001:0.0:1788573629.570703:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308989466 00000100:00000001:0.0:1788573629.570704:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308993261 00000100:00000001:0.0:1788573629.570705:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573629.570711:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.570713:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573629.570716:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573629/real 1788573629] req@ffff8fc4ac9f7100 x1875440516596608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573629.570721:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573629.570722:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573629.570724:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573629.570725:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573629.570726:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573629.570728:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573629.570729:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7100 x1875440516596608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573629.570735:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7100 x1875440516596608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570739:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573629.570740:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.570741:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573629.570742:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573629.570744:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7100 x1875440516596608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570750:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573629.570754:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7100 x1875440516596608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570761:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573629.570763:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.570765:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573629.570766:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.570768:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573629.570771:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573629.570774:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573629.570784:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573629.570786:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573629.570791:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7100 x1875440516596608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573629.570799:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516596608:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573629.570802:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.570803:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573629.570805:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516596480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570825:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573629.570826:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573629.570828:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573629.570830:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573629.570831:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573629.570832:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573629.570834:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573629.570835:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35c00. 02000000:00000001:0.0:1788573629.570837:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573629.570838:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573629.570839:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788573629.570841:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573629.570842:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.570843:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573629.570845:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7100 x1875440516596608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570849:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573629.570850:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573629.570851:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573629.570853:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573629.570854:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573629.570855:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573629.570856:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573629.570861:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36400. 02000000:00000001:0.0:1788573629.570862:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573629.570863:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573629.570864:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7100. 00000100:00000001:0.0:1788573629.570866:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573629.570867:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.570868:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.570871:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573629.570871:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.570873:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573629.570874:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573629.570875:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573629.570878:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516596736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570883:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573629.570884:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573629.570885:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573629.570886:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573629.570888:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573629.570890:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516596736:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573629.570891:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573629.570893:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573629.570894:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573629.570896:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573629.570899:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573629.570901:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573629.570905:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573629.570908:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573629.570912:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573629.570915:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516596736, portal 4 00000100:00000001:0.0:1788573629.570918:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573629.570919:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788573629.570926:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516596736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.570944:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573629.570946:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573629.570949:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788573629.570952:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516596736, offset 0 00000400:00000200:0.0:1788573629.570956:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573629.570960:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573629.570963:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573629.570965:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573629.570967:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573629.570969:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573629.570970:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573629.570973:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546314:546314:256:4294967295] 192.168.202.128@tcp LPNI seq info [546237:546237:8:4294967295] 00000400:00000200:0.0:1788573629.570975:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573629.570980:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573629.570983:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573629.570986:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788573629.570988:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573629.570990:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1788573629.570998:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573629.570999:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788573629.571001:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573629.571003:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573629.571005:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788573629.571007:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788573629.571009:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573629.571011:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516596736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571017:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573629.571019:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516596736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571031:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.571032:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573629.571035:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573629.571037:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573629.571040:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573629.571041:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308989607 00000100:00000001:0.0:1788573629.571043:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308993261 00000100:00000001:0.0:1788573629.571044:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308989607 00000100:00000001:0.0:1788573629.571046:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308993261 00000100:00000001:0.0:1788573629.571047:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573629.571048:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.571050:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573629.571052:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573629/real 1788573629] req@ffff8fc4ac9f6a00 x1875440516596736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573629.571057:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573629.571058:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573629.571059:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573629.571061:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573629.571062:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573629.571063:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573629.571065:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516596736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573629.571071:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516596736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571075:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573629.571075:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.571077:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573629.571077:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573629.571080:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516596736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571087:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573629.571090:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516596736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571094:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573629.571096:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.571097:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573629.571098:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.571099:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573629.571100:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573629.571102:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573629.571104:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573629.571105:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573629.571107:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516596736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573629.571112:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516596736:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573629.571115:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573629.571117:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4e00 x1875440516596864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571121:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573629.571122:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573629.571123:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573629.571125:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573629.571126:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573629.571128:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516596864:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573629.571129:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573629.571131:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573629.571134:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573629.571136:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573629.571137:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573629.571139:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573629.571141:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573629.571143:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573629.571145:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573629.571147:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516596864, portal 4 00000100:00000001:0.0:1788573629.571148:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573629.571149:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605324288 : -123400104227328 : ffff8fc4ac9f4e00) 00000100:00000040:0.0:1788573629.571153:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4e00 x1875440516596864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571157:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573629.571159:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573629.571160:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788573629.571162:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516596864, offset 0 00000400:00000200:0.0:1788573629.571164:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573629.571166:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573629.571169:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573629.571170:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573629.571172:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573629.571173:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573629.571175:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573629.571177:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546315:546315:256:4294967295] 192.168.202.128@tcp LPNI seq info [546238:546238:8:4294967295] 00000400:00000200:0.0:1788573629.571180:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573629.571184:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573629.571187:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573629.571189:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788573629.571192:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573629.571194:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1788573629.571205:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573629.571209:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788573629.571211:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573629.571213:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573629.571215:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788573629.571217:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788573629.571218:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573629.571220:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4e00 x1875440516596864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571226:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573629.571228:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4e00 x1875440516596864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573654 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571232:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.571233:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573629.571235:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573629.571237:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573629.571239:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573629.571240:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308989609 00000100:00000001:0.0:1788573629.571242:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308993261 00000100:00000001:0.0:1788573629.571243:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308989609 00000100:00000001:0.0:1788573629.571245:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308993261 00000100:00000001:0.0:1788573629.571246:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573629.571247:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.571248:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573629.571251:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573629/real 1788573629] req@ffff8fc4ac9f4e00 x1875440516596864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573629.571255:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573629.571256:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573629.571257:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573629.571259:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573629.571260:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573629.571264:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573629.571267:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4e00 x1875440516596864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573629.571272:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4e00 x1875440516596864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571276:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573629.571277:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.571278:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573629.571279:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573629.571281:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4e00 x1875440516596864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571285:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573629.571287:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4e00 x1875440516596864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571292:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573629.571293:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.571294:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573629.571295:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.571296:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573629.571297:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573629.571299:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573629.571301:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573629.571302:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573629.571304:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4e00 x1875440516596864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573629.571309:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516596864:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573629.571311:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.571312:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573629.571317:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516596736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571321:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573629.571322:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573629.571324:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573629.571325:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573629.571326:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573629.571327:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573629.571329:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573629.571330:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34c00. 02000000:00000001:0.0:1788573629.571332:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573629.571332:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573629.571333:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788573629.571335:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573629.571336:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.571337:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573629.571339:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4e00 x1875440516596864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573654 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573629.571343:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573629.571344:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573629.571345:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573629.571346:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573629.571347:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573629.571347:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573629.571348:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573629.571349:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37c00. 02000000:00000001:0.0:1788573629.571351:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573629.571351:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573629.571352:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4e00. 00000100:00000001:0.0:1788573629.571353:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573629.571354:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.571356:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573629.571358:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573629.571359:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573629.571360:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573629.571361:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573630.080901:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573630.080907:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573630.080912:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573630.080919:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573630.080926:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573630.080932:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573630.080942:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573630.080953:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573630.080958:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573631.104914:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573631.104922:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573631.104928:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573631.104937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573631.104947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573631.104954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573631.104976:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573631.104981:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573631.104985:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573632.128922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573632.128930:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573632.128932:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573632.128939:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573632.128944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573632.128952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573632.128958:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573632.128962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573632.128968:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573633.152949:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573633.152958:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573633.152963:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573633.152971:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573633.152979:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573633.152986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573633.153000:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573633.153003:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573633.153007:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573633.153047:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573633.153050:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573633.153058:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573633.153061:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573633.153063:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573633.153065:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573633.153066:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573633.153067:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573633.153069:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573633.408884:0:265627:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308993261, at 4308993280 00000100:00000010:0.0:1788573633.408904:0:265627:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573633.857048:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573633.857055:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573633.857060:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573634.176992:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573634.177008:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573634.177013:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573634.177023:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573634.177037:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573634.177050:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573634.177057:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573634.177068:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573634.177076:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573634.689284:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573634.689291:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573634.689296:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573634.689320:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7b80 x1875440516596992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573634.689342:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573634.689346:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573634.689351:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573634.689357:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573634.689364:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573634.689375:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516596992:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573634.689385:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573634.689392:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573634.689397:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573634.689410:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573634.689416:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573634.689423:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573634.689435:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573634.689451:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573634.689463:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573634.689484:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516596992, portal 25 00000100:00000001:0.0:1788573634.689491:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573634.689494:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335936 : -123400104215680 : ffff8fc4ac9f7b80) 00000100:00000040:0.0:1788573634.689505:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7b80 x1875440516596992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573634.689523:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573634.689531:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573634.689537:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788573634.689545:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516596992, offset 0 00000400:00000200:0.0:1788573634.689557:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573634.689572:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573634.689584:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573634.689592:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573634.689597:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573634.689601:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573634.689605:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573634.689609:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546316:546316:256:4294967295] 192.168.202.128@tcp LPNI seq info [546239:546239:8:4294967295] 00000400:00000200:0.0:1788573634.689615:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573634.689627:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573634.689638:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573634.689647:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788573634.689662:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573634.689703:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573634.689713:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573634.689719:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4308998381 00000100:00000010:0.0:1788573634.689729:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573634.689753:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573634.689764:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4308998381 00000100:00000010:0.0:1788573634.689772:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50ea0. 00000100:00000001:0.0:1788573634.689780:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573634.689788:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573634.689795:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573634.689802:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573634.689807:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573634.689844:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573634.689848:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573634.689852:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573634.689856:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573634.689859:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573634.689868:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5880 x1875440516597120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573634.689888:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573634.689893:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573634.689898:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573634.689902:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573634.689909:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573634.689918:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516597120:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573634.689927:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573634.689934:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573634.689938:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573634.689943:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573634.689949:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573634.689952:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573634.689959:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573634.689966:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915790. 00000400:00000010:0.0:1788573634.689974:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573634.689981:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516597120, portal 10 00000100:00000001:0.0:1788573634.689987:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573634.689991:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605326976 : -123400104224640 : ffff8fc4ac9f5880) 00000100:00000040:0.0:1788573634.690014:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5880 x1875440516597120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573634.690034:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573634.690040:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573634.690046:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9330. 00000100:00000200:0.0:1788573634.690053:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516597120, offset 0 00000400:00000200:0.0:1788573634.690064:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573634.690073:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573634.690082:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573634.690088:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573634.690095:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573634.690101:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573634.690106:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573634.690115:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546317:546317:255:4294967295] 192.168.202.128@tcp LPNI seq info [546240:546240:7:4294967295] 00000400:00000200:0.0:1788573634.690125:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573634.690136:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573634.690145:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573634.690153:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56200. 00000800:00000200:0.0:1788573634.690163:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573634.690178:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573634.690184:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573634.690194:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573634.690199:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308994579 00000100:00000001:0.0:1788573634.690205:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308998381 00000100:00000001:0.0:1788573634.690211:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308994579 00000100:00000001:0.0:1788573634.690216:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308998381 00000100:00000001:0.0:1788573634.690221:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573634.690226:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573634.690232:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573634.690243:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573634.690251:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573634.690254:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573634.690260:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573634.690264:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573634.690269:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573634.690279:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6d80 x1875440516597248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573634.690299:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573634.690302:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573634.690321:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573634.690325:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573634.690332:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573634.690342:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516597248:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573634.690350:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573634.690355:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573634.690357:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573634.690362:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573634.690369:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573634.690374:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573634.690382:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573634.690388:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915948. 00000400:00000010:0.0:1788573634.690396:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9908. 00000100:00000200:0.0:1788573634.690403:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516597248, portal 10 00000100:00000001:0.0:1788573634.690410:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573634.690412:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605332352 : -123400104219264 : ffff8fc4ac9f6d80) 00000100:00000040:0.0:1788573634.690424:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6d80 x1875440516597248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573634.690440:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573634.690448:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573634.690453:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9c38. 00000100:00000200:0.0:1788573634.690458:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516597248, offset 0 00000400:00000200:0.0:1788573634.690468:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573634.690478:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573634.690492:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573634.690498:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573634.690504:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573634.690509:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573634.690516:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573634.690524:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546318:546318:254:4294967295] 192.168.202.128@tcp LPNI seq info [546241:546241:6:4294967295] 00000400:00000200:0.0:1788573634.690535:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573634.690552:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573634.690567:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573634.690573:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573634.690580:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573634.690599:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573634.690603:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573634.690607:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573634.690610:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308994586 00000100:00000001:0.0:1788573634.690614:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308998381 00000100:00000001:0.0:1788573634.690617:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308994586 00000100:00000001:0.0:1788573634.690620:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308998381 00000100:00000001:0.0:1788573634.690623:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573634.690626:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573634.690629:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573634.690633:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5c00 x1875440516597376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573634.690641:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573634.690643:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573634.690645:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573634.690647:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573634.690650:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573634.690653:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516597376:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573634.690665:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573634.690668:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573634.690669:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573634.690671:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573634.690677:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573634.690679:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573634.690684:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573634.690687:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573634.690698:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573634.690702:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516597376, portal 4 00000100:00000001:0.0:1788573634.690705:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573634.690706:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605327872 : -123400104223744 : ffff8fc4ac9f5c00) 00000100:00000040:0.0:1788573634.690714:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5c00 x1875440516597376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573634.690721:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573634.690724:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573634.690727:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9e58. 00000100:00000200:0.0:1788573634.690730:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516597376, offset 0 00000400:00000200:0.0:1788573634.690735:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573634.690740:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573634.690744:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573634.690752:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573634.690755:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573634.690758:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573634.690761:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573634.690764:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546319:546319:253:4294967295] 192.168.202.128@tcp LPNI seq info [546242:546242:5:4294967295] 00000400:00000200:0.0:1788573634.690769:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573634.690776:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573634.690782:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573634.690786:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56f00. 00000800:00000200:0.0:1788573634.690791:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573634.690805:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573634.690833:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573634.690839:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573634.690842:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308994727 00000100:00000001:0.0:1788573634.690844:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308998381 00000100:00000001:0.0:1788573634.690847:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308994727 00000100:00000001:0.0:1788573634.690849:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308998381 00000100:00000001:0.0:1788573634.690851:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573634.690854:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573634.690856:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573634.690858:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573634.690862:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573634.690863:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573634.690866:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573634.690867:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573634.690869:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573634.690873:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5500 x1875440516597504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573634.690882:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573634.690884:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573634.690886:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573634.690888:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573634.690890:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573634.690894:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516597504:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573634.690898:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573634.690900:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573634.690902:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573634.690904:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573634.690908:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573634.690911:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573634.690915:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573634.690918:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573634.690928:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed97f8. 00000100:00000200:0.0:1788573634.690931:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516597504, portal 4 00000100:00000001:0.0:1788573634.690934:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573634.690936:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605326080 : -123400104225536 : ffff8fc4ac9f5500) 00000100:00000040:0.0:1788573634.690943:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5500 x1875440516597504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573634.690951:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573634.690953:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573634.690957:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9d48. 00000100:00000200:0.0:1788573634.690959:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516597504, offset 0 00000400:00000200:0.0:1788573634.690964:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573634.690969:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573634.690974:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573634.690977:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573634.690980:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573634.690982:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573634.690985:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573634.690989:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546320:546320:252:4294967295] 192.168.202.128@tcp LPNI seq info [546243:546243:4:4294967295] 00000400:00000200:0.0:1788573634.690994:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573634.691000:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573634.691006:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573634.691010:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788573634.691015:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573634.691022:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573634.691026:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573634.691030:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573634.691032:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308994729 00000100:00000001:0.0:1788573634.691035:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4308998381 00000100:00000001:0.0:1788573634.691037:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308994729 00000100:00000001:0.0:1788573634.691039:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4308998381 00000100:00000001:0.0:1788573634.691051:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573634.691054:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573634.691057:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573634.691060:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573634.691064:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573634.691066:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573634.691068:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573634.691069:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573634.691071:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573634.691089:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573635.201032:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573635.201039:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573635.201045:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573635.201055:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573635.201072:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573635.201076:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573635.201080:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573635.201083:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573635.201086:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573636.224958:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573636.224972:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573636.224987:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573636.225004:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573636.225021:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573636.225035:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573636.225074:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573636.225079:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573636.225086:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573637.248954:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573637.248965:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573637.248970:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573637.248977:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573637.248982:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573637.248993:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.249004:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573637.249012:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.249018:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573637.761694:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573637.761709:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573637.761713:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573637.761723:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573634/real 1788573637] req@ffff8fc4ac9f5500 x1875440516597504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573637.761739:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573637.761742:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573637.761747:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573637.761752:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed97f8 00000400:00000010:0.0:1788573637.761754:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed97f8. 00000100:00000001:0.0:1788573637.761757:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573637.761761:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5500 x1875440516597504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573637.761770:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5500 x1875440516597504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.761778:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573637.761781:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.761785:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573637.761789:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573637.761793:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5500 x1875440516597504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.761801:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573637.761849:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5500 x1875440516597504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.761858:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573637.761862:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.761864:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573637.761866:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.761870:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573637.761874:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573637.761879:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573637.761894:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573637.761898:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573637.761904:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5500 x1875440516597504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573637.761912:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516597504:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573637.761920:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573637.761924:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573634/real 1788573637] req@ffff8fc4ac9f6d80 x1875440516597248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573637.761934:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573637.761936:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573637.761939:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915948. 00000400:00000200:0.0:1788573637.761942:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9908 00000400:00000010:0.0:1788573637.761945:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9908. 00000100:00000001:0.0:1788573637.761947:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573637.761950:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6d80 x1875440516597248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573637.761959:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6d80 x1875440516597248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.761966:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573637.761968:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.761969:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573637.761971:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573637.761976:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6d80 x1875440516597248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.761983:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573637.761987:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6d80 x1875440516597248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762006:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573637.762008:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762011:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573637.762012:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762015:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573637.762019:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573637.762022:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573637.762027:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573637.762029:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573637.762033:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6d80 x1875440516597248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573637.762042:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516597248:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573637.762047:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573637.762051:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573634/real 1788573637] req@ffff8fc4ac9f5c00 x1875440516597376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573637.762060:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573637.762062:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573637.762064:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573637.762067:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573637.762069:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573637.762072:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573637.762076:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5c00 x1875440516597376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573637.762085:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5c00 x1875440516597376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762091:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573637.762099:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762101:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573637.762103:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573637.762108:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5c00 x1875440516597376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762116:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573637.762120:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5c00 x1875440516597376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762128:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573637.762130:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762132:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573637.762134:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762136:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573637.762138:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573637.762141:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573637.762145:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573637.762148:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573637.762152:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5c00 x1875440516597376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573637.762160:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516597376:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573637.762165:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573637.762170:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573634/real 1788573637] req@ffff8fc4ac9f5880 x1875440516597120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573637.762179:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573637.762181:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573637.762183:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915790. 00000400:00000200:0.0:1788573637.762186:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573637.762188:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573637.762196:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573637.762200:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5880 x1875440516597120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573637.762207:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5880 x1875440516597120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762215:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573637.762217:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762219:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573637.762221:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573637.762225:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5880 x1875440516597120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762232:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573637.762236:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5880 x1875440516597120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762245:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573637.762248:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762250:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573637.762251:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762253:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573637.762255:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573637.762257:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573637.762279:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573637.762282:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573637.762287:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5880 x1875440516597120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573637.762295:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516597120:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573637.762300:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573637.762312:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573634/real 1788573637] req@ffff8fc4ac9f7b80 x1875440516596992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573637.762322:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573637.762323:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573637.762326:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573637.762329:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573637.762332:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573637.762335:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573637.762339:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7b80 x1875440516596992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573637.762348:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7b80 x1875440516596992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762355:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573637.762357:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762359:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573637.762360:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573637.762363:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7b80 x1875440516596992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762371:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573637.762376:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7b80 x1875440516596992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762384:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573637.762386:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762388:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573637.762389:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762391:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573637.762393:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573637.762396:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573637.762401:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573637.762403:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573637.762419:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7b80 x1875440516596992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573637.762428:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516596992:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573637.762439:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762442:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573637.762446:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5500 x1875440516597504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762453:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573637.762456:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573637.762461:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573637.762466:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573637.762470:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573637.762472:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573637.762475:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573637.762478:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35800. 02000000:00000001:0.0:1788573637.762483:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573637.762485:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573637.762488:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5500. 00000100:00000001:0.0:1788573637.762492:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573637.762494:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762496:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573637.762500:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6d80 x1875440516597248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762508:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573637.762510:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573637.762512:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573637.762515:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573637.762516:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573637.762518:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573637.762520:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573637.762521:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788573637.762523:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573637.762530:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573637.762531:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6d80. 00000100:00000001:0.0:1788573637.762535:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573637.762536:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762538:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573637.762542:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5c00 x1875440516597376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762549:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573637.762552:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573637.762554:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573637.762556:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573637.762558:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573637.762559:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573637.762561:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573637.762563:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788573637.762566:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573637.762567:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573637.762569:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5c00. 00000100:00000001:0.0:1788573637.762571:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573637.762573:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762574:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573637.762578:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5880 x1875440516597120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762585:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573637.762586:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573637.762588:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573637.762590:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573637.762592:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573637.762593:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573637.762595:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573637.762597:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788573637.762600:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573637.762601:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573637.762602:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5880. 00000100:00000001:0.0:1788573637.762605:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573637.762607:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762615:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573637.762619:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7b80 x1875440516596992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573659 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573637.762626:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573637.762628:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573637.762629:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573637.762648:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573637.762650:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573637.762651:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573637.762654:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573637.762656:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788573637.762659:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573637.762660:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573637.762662:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7b80. 00000100:00000001:0.0:1788573637.762674:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573637.762675:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762678:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573637.762688:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573637.762690:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573637.762692:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573637.762694:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573638.272923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573638.272933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573638.272939:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573638.272946:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573638.272952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573638.272957:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573638.272972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573638.272976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573638.272981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573638.528966:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4308998381, at 4308998400 00000100:00000010:0.0:1788573638.528994:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573638.976992:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573638.976998:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573638.977002:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573639.297020:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573639.297038:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573639.297044:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573639.297048:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573639.297055:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573639.297066:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.297073:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573639.297081:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.297087:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573639.810106:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573639.810113:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573639.810116:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573639.810123:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516597632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.810133:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573639.810137:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573639.810142:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573639.810146:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573639.810150:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573639.810155:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516597632:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573639.810160:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573639.810166:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573639.810170:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573639.810174:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573639.810181:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573639.810183:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573639.810191:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573639.810196:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573639.810204:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573639.810215:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516597632, portal 25 00000100:00000001:0.0:1788573639.810218:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573639.810220:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788573639.810227:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516597632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.810246:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573639.810257:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573639.810261:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573639.810264:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516597632, offset 0 00000400:00000200:0.0:1788573639.810275:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573639.810286:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573639.810294:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573639.810302:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573639.810307:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573639.810311:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573639.810314:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573639.810319:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546321:546321:256:4294967295] 192.168.202.128@tcp LPNI seq info [546244:546244:8:4294967295] 00000400:00000200:0.0:1788573639.810324:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573639.810337:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573639.810345:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573639.810352:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573639.810361:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573639.810366:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573639.810380:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573639.810383:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573639.810390:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573639.810395:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573639.810403:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573639.810406:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573639.810414:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573639.810419:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516597632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.810431:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573639.810435:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516597632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.810444:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573639.810446:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573639.810460:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573639.810466:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573639.810470:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309003502 00000100:00000010:0.0:1788573639.810475:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573639.810485:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573639.810493:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309003502 00000100:00000010:0.0:1788573639.810497:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b508a0. 00000100:00000001:0.0:1788573639.810501:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573639.810505:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573639.810509:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573639.810512:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573639.810518:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573639/real 1788573639] req@ffff8fc4ac822a00 x1875440516597632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573639.810532:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573639.810534:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573639.810537:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573639.810541:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573639.810542:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573639.810545:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573639.810550:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516597632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573639.810559:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516597632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.810568:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573639.810570:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.810573:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573639.810577:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573639.810582:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516597632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.810590:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573639.810595:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516597632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.810609:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573639.810613:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.810615:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573639.810616:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.810619:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573639.810622:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573639.810631:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573639.810637:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573639.810640:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573639.810645:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516597632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573639.810654:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516597632:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573639.810659:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573639.810662:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516597760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.810668:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573639.810669:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573639.810672:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573639.810674:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573639.810677:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573639.810680:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516597760:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573639.810683:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573639.810685:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573639.810687:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573639.810689:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573639.810693:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573639.810695:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573639.810699:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573639.810709:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573639.810713:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573639.810717:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516597760, portal 10 00000100:00000001:0.0:1788573639.810719:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573639.810721:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788573639.810729:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516597760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.810736:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573639.810738:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573639.810740:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573639.810743:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516597760, offset 0 00000400:00000200:0.0:1788573639.810747:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573639.810751:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573639.810755:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573639.810758:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573639.810761:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573639.810763:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573639.810766:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573639.810770:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546322:546322:256:4294967295] 192.168.202.128@tcp LPNI seq info [546245:546245:8:4294967295] 00000400:00000200:0.0:1788573639.810774:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573639.810780:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573639.810785:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573639.810789:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573639.810793:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573639.810796:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573639.810863:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573639.810865:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573639.810870:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573639.810873:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573639.810877:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573639.810879:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573639.810890:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573639.810895:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516597760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.810904:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573639.810908:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516597760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.810915:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573639.810917:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573639.810922:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573639.810927:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573639.810931:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573639.810933:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308999700 00000100:00000001:0.0:1788573639.810937:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309003502 00000100:00000001:0.0:1788573639.810939:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308999700 00000100:00000001:0.0:1788573639.810942:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309003502 00000100:00000001:0.0:1788573639.810944:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573639.810947:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573639.810949:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573639.810953:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573639/real 1788573639] req@ffff8fc4ac823100 x1875440516597760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573639.810962:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573639.810964:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573639.810966:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573639.810969:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573639.810971:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573639.810973:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573639.810977:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516597760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573639.810986:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516597760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811005:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573639.811007:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.811009:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573639.811011:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573639.811015:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516597760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811023:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573639.811027:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516597760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811034:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573639.811036:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.811038:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573639.811039:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.811041:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573639.811043:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573639.811046:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573639.811049:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573639.811052:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573639.811056:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516597760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573639.811064:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516597760:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573639.811069:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573639.811073:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516597888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811080:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573639.811081:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573639.811092:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573639.811094:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573639.811096:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573639.811100:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516597888:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573639.811104:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573639.811106:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573639.811108:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573639.811110:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573639.811114:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573639.811122:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573639.811127:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573639.811130:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573639.811135:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573639.811138:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516597888, portal 10 00000100:00000001:0.0:1788573639.811141:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573639.811143:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788573639.811149:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516597888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811156:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573639.811159:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573639.811162:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573639.811164:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516597888, offset 0 00000400:00000200:0.0:1788573639.811169:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573639.811173:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573639.811177:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573639.811180:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573639.811183:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573639.811186:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573639.811188:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573639.811191:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546323:546323:256:4294967295] 192.168.202.128@tcp LPNI seq info [546246:546246:8:4294967295] 00000400:00000200:0.0:1788573639.811195:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573639.811202:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573639.811214:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573639.811217:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573639.811221:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573639.811224:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573639.811253:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573639.811255:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573639.811258:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573639.811262:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573639.811266:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573639.811268:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573639.811271:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573639.811275:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516597888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811283:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573639.811286:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516597888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811293:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573639.811295:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573639.811298:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573639.811302:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573639.811306:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573639.811308:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308999707 00000100:00000001:0.0:1788573639.811311:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309003502 00000100:00000001:0.0:1788573639.811313:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308999707 00000100:00000001:0.0:1788573639.811315:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309003502 00000100:00000001:0.0:1788573639.811323:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573639.811326:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573639.811328:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573639.811332:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573639/real 1788573639] req@ffff8fc4ac821500 x1875440516597888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573639.811347:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573639.811349:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573639.811351:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573639.811354:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573639.811356:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573639.811358:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573639.811362:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516597888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573639.811370:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516597888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811377:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573639.811379:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.811381:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573639.811382:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573639.811386:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516597888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811393:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573639.811397:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516597888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811410:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573639.811413:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.811415:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573639.811416:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.811419:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573639.811422:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573639.811425:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573639.811429:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573639.811433:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573639.811443:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516597888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573639.811453:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516597888:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573639.811457:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573639.811462:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516598016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811469:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573639.811470:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573639.811473:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573639.811475:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573639.811477:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573639.811480:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516598016:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573639.811484:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573639.811486:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573639.811487:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573639.811489:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573639.811492:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573639.811495:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573639.811499:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573639.811502:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573639.811505:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573639.811508:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516598016, portal 4 00000100:00000001:0.0:1788573639.811510:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573639.811512:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788573639.811518:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516598016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811524:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573639.811528:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573639.811531:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573639.811534:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516598016, offset 0 00000400:00000200:0.0:1788573639.811548:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573639.811553:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573639.811557:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573639.811560:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573639.811563:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573639.811565:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573639.811568:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573639.811572:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546324:546324:256:4294967295] 192.168.202.128@tcp LPNI seq info [546247:546247:8:4294967295] 00000400:00000200:0.0:1788573639.811576:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573639.811583:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573639.811588:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573639.811591:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573639.811595:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573639.811598:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573639.811610:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573639.811612:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573639.811615:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573639.811619:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573639.811622:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573639.811624:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573639.811627:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573639.811631:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516598016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811640:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573639.811644:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516598016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811650:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573639.811652:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573639.811655:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573639.811658:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573639.811662:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573639.811670:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308999847 00000100:00000001:0.0:1788573639.811673:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309003502 00000100:00000001:0.0:1788573639.811675:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308999847 00000100:00000001:0.0:1788573639.811678:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309003502 00000100:00000001:0.0:1788573639.811680:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573639.811683:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573639.811685:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573639.811689:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573639/real 1788573639] req@ffff8fc4ac823b80 x1875440516598016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573639.811698:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573639.811700:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573639.811702:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573639.811704:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573639.811706:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573639.811708:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573639.811712:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516598016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573639.811721:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516598016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811728:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573639.811730:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.811732:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573639.811733:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573639.811737:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516598016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811744:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573639.811749:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516598016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811756:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573639.811763:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.811765:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573639.811767:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.811769:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573639.811771:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573639.811774:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573639.811778:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573639.811781:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573639.811785:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516598016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573639.811794:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516598016:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573639.811799:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573639.811802:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516598144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811831:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573639.811833:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573639.811836:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573639.811838:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573639.811840:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573639.811843:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516598144:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573639.811846:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573639.811848:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573639.811850:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573639.811852:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573639.811856:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573639.811858:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573639.811862:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573639.811865:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573639.811868:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573639.811871:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516598144, portal 4 00000100:00000001:0.0:1788573639.811878:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573639.811879:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788573639.811887:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516598144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.811894:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573639.811896:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573639.811899:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573639.811902:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516598144, offset 0 00000400:00000200:0.0:1788573639.811906:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573639.811911:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573639.811915:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573639.811918:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573639.811921:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573639.811923:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573639.811926:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573639.811929:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546325:546325:256:4294967295] 192.168.202.128@tcp LPNI seq info [546248:546248:8:4294967295] 00000400:00000200:0.0:1788573639.811934:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573639.811941:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573639.811946:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573639.811950:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573639.811954:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573639.811957:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573639.811969:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573639.811971:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573639.811975:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573639.811978:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573639.811982:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573639.811984:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573639.811986:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573639.811990:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516598144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.812012:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573639.812016:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516598144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.812023:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573639.812025:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573639.812029:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573639.812033:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573639.812036:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573639.812038:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4308999850 00000100:00000001:0.0:1788573639.812041:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309003502 00000100:00000001:0.0:1788573639.812043:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4308999850 00000100:00000001:0.0:1788573639.812046:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309003502 00000100:00000001:0.0:1788573639.812048:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573639.812050:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573639.812052:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573639.812057:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573639/real 1788573639] req@ffff8fc4ac820000 x1875440516598144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573639.812065:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573639.812067:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573639.812069:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573639.812072:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573639.812074:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573639.812076:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573639.812080:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516598144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573639.812090:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516598144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.812097:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573639.812104:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.812106:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573639.812108:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573639.812112:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516598144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.812120:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573639.812124:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516598144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.812132:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573639.812134:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.812136:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573639.812137:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.812140:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573639.812142:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573639.812144:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573639.812148:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573639.812151:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573639.812156:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516598144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573639.812164:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516598144:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573639.812169:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.812172:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573639.812176:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516597632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.812184:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573639.812186:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573639.812190:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573639.812195:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573639.812198:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573639.812206:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573639.812209:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573639.812212:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788573639.812217:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573639.812220:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573639.812222:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788573639.812226:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573639.812227:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.812229:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573639.812249:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516597760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.812257:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573639.812258:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573639.812260:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573639.812263:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573639.812265:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573639.812266:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573639.812269:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573639.812270:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788573639.812273:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573639.812274:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573639.812276:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788573639.812279:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573639.812280:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.812282:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573639.812285:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516597888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.812292:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573639.812294:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573639.812295:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573639.812298:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573639.812299:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573639.812301:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573639.812303:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573639.812304:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788573639.812307:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573639.812316:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573639.812318:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788573639.812321:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573639.812323:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.812325:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573639.812329:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516598016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.812336:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573639.812338:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573639.812340:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573639.812343:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573639.812344:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573639.812346:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573639.812348:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573639.812350:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788573639.812354:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573639.812355:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573639.812356:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788573639.812359:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573639.812361:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.812363:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573639.812366:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516598144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573664 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573639.812374:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573639.812375:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573639.812377:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573639.812379:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573639.812380:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573639.812382:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573639.812384:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573639.812386:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788573639.812388:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573639.812390:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573639.812391:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788573639.812394:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573639.812395:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.812404:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573639.812414:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573639.812416:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573639.812419:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573639.812421:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573640.321003:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573640.321015:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573640.321019:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573640.321032:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573640.321045:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573640.321055:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573640.321085:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573640.321091:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573640.321098:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573641.345022:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573641.345031:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573641.345038:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573641.345048:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573641.345056:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573641.345062:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573641.345079:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573641.345083:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573641.345088:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573642.369008:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573642.369016:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573642.369026:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573642.369039:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573642.369050:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573642.369055:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573642.369064:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573642.369067:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573642.369071:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573643.393067:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573643.393074:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573643.393079:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573643.393083:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573643.393086:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573643.393089:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573643.393099:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573643.393102:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573643.393105:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573643.648928:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309003502, at 4309003520 00000100:00000010:0.0:1788573643.648950:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573644.097031:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573644.097034:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573644.097038:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573644.416985:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573644.416993:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573644.417002:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573644.417017:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573644.417032:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573644.417046:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.417076:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573644.417083:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.417092:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.929186:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573644.929199:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573644.929208:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573644.929234:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516598272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.929285:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573644.929291:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573644.929301:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573644.929307:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573644.929313:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573644.929327:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516598272:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573644.929344:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573644.929354:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573644.929361:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573644.929367:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573644.929377:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573644.929386:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573644.929407:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573644.929432:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573644.929447:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573644.929459:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516598272, portal 25 00000100:00000001:0.0:1788573644.929470:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573644.929475:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788573644.929494:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516598272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.929534:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573644.929550:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573644.929560:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573644.929567:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516598272, offset 0 00000400:00000200:0.0:1788573644.929594:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573644.929610:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573644.929629:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573644.929650:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573644.929667:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573644.929681:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573644.929693:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573644.929713:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546326:546326:256:4294967295] 192.168.202.128@tcp LPNI seq info [546249:546249:8:4294967295] 00000400:00000200:0.0:1788573644.929737:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573644.929763:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573644.929780:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573644.929790:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573644.929846:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573644.929861:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573644.929906:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573644.929912:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573644.929921:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573644.929941:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573644.929959:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573644.929965:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573644.929981:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573644.930005:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516598272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930057:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573644.930069:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516598272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930096:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.930102:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573644.930111:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573644.930126:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573644.930132:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309008622 00000100:00000010:0.0:1788573644.930145:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573644.930171:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573644.930182:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309008622 00000100:00000010:0.0:1788573644.930191:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b507e0. 00000100:00000001:0.0:1788573644.930197:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573644.930208:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573644.930217:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.930223:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573644.930232:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573644/real 1788573644] req@ffff8fc4ac820000 x1875440516598272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573644.930260:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573644.930264:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573644.930269:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573644.930276:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573644.930282:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573644.930289:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573644.930299:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516598272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573644.930325:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516598272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930356:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573644.930359:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.930363:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573644.930366:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573644.930372:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516598272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930380:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573644.930385:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516598272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930392:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573644.930395:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.930397:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573644.930398:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.930401:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573644.930404:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573644.930408:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573644.930412:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573644.930415:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573644.930420:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516598272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573644.930428:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516598272:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573644.930433:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.930435:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573644.930439:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516598272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930445:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573644.930448:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573644.930456:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573644.930459:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573644.930462:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573644.930464:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573644.930467:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573644.930469:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862ac00. 02000000:00000001:0.0:1788573644.930474:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573644.930476:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573644.930479:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788573644.930482:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573644.930484:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.930486:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.930494:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573644.930496:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.930499:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573644.930501:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573644.930503:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573644.930508:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516598400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930516:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573644.930518:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573644.930520:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573644.930523:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573644.930525:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573644.930528:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516598400:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573644.930532:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573644.930535:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573644.930537:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573644.930538:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573644.930542:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573644.930544:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573644.930549:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573644.930552:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573644.930555:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573644.930559:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516598400, portal 10 00000100:00000001:0.0:1788573644.930568:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573644.930570:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788573644.930577:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516598400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930584:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573644.930586:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573644.930589:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573644.930592:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516598400, offset 0 00000400:00000200:0.0:1788573644.930596:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573644.930601:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573644.930606:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573644.930610:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573644.930612:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573644.930615:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573644.930617:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573644.930621:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546327:546327:256:4294967295] 192.168.202.128@tcp LPNI seq info [546250:546250:8:4294967295] 00000400:00000200:0.0:1788573644.930625:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573644.930632:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573644.930637:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573644.930641:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573644.930644:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573644.930648:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573644.930660:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573644.930662:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573644.930665:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573644.930668:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573644.930671:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573644.930673:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573644.930676:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573644.930680:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516598400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930699:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573644.930703:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516598400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930710:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.930711:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573644.930715:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573644.930719:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573644.930723:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573644.930725:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309004819 00000100:00000001:0.0:1788573644.930727:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309008622 00000100:00000001:0.0:1788573644.930729:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309004819 00000100:00000001:0.0:1788573644.930731:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309008622 00000100:00000001:0.0:1788573644.930734:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573644.930736:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.930738:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573644.930742:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573644/real 1788573644] req@ffff8fc4ac823b80 x1875440516598400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573644.930751:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573644.930752:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573644.930754:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573644.930756:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573644.930757:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573644.930760:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573644.930763:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516598400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573644.930771:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516598400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930778:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573644.930785:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.930787:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573644.930788:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573644.930793:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516598400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930800:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573644.930804:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516598400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930832:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573644.930834:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.930836:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573644.930837:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.930839:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573644.930841:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573644.930844:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573644.930848:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573644.930850:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573644.930855:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516598400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573644.930863:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516598400:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573644.930868:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573644.930871:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516598528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930878:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573644.930880:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573644.930882:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573644.930884:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573644.930886:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573644.930895:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516598528:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573644.930899:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573644.930901:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573644.930903:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573644.930904:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573644.930907:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573644.930909:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573644.930913:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573644.930916:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573644.930919:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573644.930922:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516598528, portal 10 00000100:00000001:0.0:1788573644.930924:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573644.930926:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788573644.930932:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516598528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.930938:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573644.930941:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573644.930943:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573644.930945:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516598528, offset 0 00000400:00000200:0.0:1788573644.930950:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573644.930954:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573644.930957:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573644.930960:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573644.930962:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573644.930965:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573644.930967:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573644.930971:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546328:546328:256:4294967295] 192.168.202.128@tcp LPNI seq info [546251:546251:8:4294967295] 00000400:00000200:0.0:1788573644.930975:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573644.930981:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573644.930986:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573644.930989:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573644.930998:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573644.931001:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573644.931012:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573644.931014:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573644.931017:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573644.931021:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573644.931025:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573644.931026:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573644.931029:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573644.931032:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516598528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931040:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573644.931043:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516598528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931050:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.931051:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573644.931055:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573644.931058:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573644.931062:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573644.931064:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309004827 00000100:00000001:0.0:1788573644.931067:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309008622 00000100:00000001:0.0:1788573644.931069:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309004827 00000100:00000001:0.0:1788573644.931071:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309008622 00000100:00000001:0.0:1788573644.931073:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573644.931075:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.931077:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573644.931081:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573644/real 1788573644] req@ffff8fc4ac821500 x1875440516598528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573644.931089:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573644.931095:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573644.931097:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573644.931100:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573644.931101:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573644.931104:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573644.931109:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516598528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573644.931117:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516598528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931123:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573644.931126:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931128:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573644.931129:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573644.931133:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516598528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931140:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573644.931144:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516598528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931151:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573644.931154:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931155:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573644.931156:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931172:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573644.931176:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573644.931178:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573644.931182:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573644.931184:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573644.931188:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516598528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573644.931208:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516598528:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573644.931213:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573644.931217:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516598656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931224:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573644.931226:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573644.931228:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573644.931230:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573644.931232:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573644.931235:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516598656:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573644.931238:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573644.931240:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573644.931242:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573644.931244:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573644.931248:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573644.931250:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573644.931254:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573644.931257:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573644.931261:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573644.931264:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516598656, portal 4 00000100:00000001:0.0:1788573644.931266:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573644.931267:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788573644.931273:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516598656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931280:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573644.931283:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573644.931285:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573644.931287:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516598656, offset 0 00000400:00000200:0.0:1788573644.931292:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573644.931302:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573644.931307:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573644.931310:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573644.931313:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573644.931315:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573644.931317:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573644.931321:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546329:546329:256:4294967295] 192.168.202.128@tcp LPNI seq info [546252:546252:8:4294967295] 00000400:00000200:0.0:1788573644.931325:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573644.931331:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573644.931336:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573644.931340:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573644.931344:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573644.931347:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573644.931358:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573644.931360:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573644.931363:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573644.931367:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573644.931370:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573644.931372:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573644.931375:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573644.931379:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516598656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931387:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573644.931391:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516598656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931397:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.931399:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573644.931402:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573644.931405:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573644.931408:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573644.931416:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309004967 00000100:00000001:0.0:1788573644.931419:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309008622 00000100:00000001:0.0:1788573644.931421:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309004967 00000100:00000001:0.0:1788573644.931423:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309008622 00000100:00000001:0.0:1788573644.931425:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573644.931428:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.931430:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573644.931434:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573644/real 1788573644] req@ffff8fc4ac823100 x1875440516598656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573644.931442:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573644.931444:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573644.931446:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573644.931448:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573644.931450:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573644.931452:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573644.931456:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516598656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573644.931464:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516598656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931470:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573644.931471:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931473:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573644.931475:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573644.931478:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516598656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931486:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573644.931490:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516598656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931497:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573644.931504:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931506:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573644.931508:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931509:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573644.931511:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573644.931514:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573644.931518:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573644.931520:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573644.931525:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516598656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573644.931533:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516598656:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573644.931536:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573644.931540:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516598784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931548:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573644.931550:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573644.931552:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573644.931554:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573644.931556:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573644.931559:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516598784:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573644.931562:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573644.931564:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573644.931566:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573644.931568:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573644.931571:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573644.931573:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573644.931577:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573644.931579:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573644.931583:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573644.931586:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516598784, portal 4 00000100:00000001:0.0:1788573644.931593:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573644.931595:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788573644.931601:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516598784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931607:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573644.931609:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573644.931611:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573644.931614:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516598784, offset 0 00000400:00000200:0.0:1788573644.931618:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573644.931622:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573644.931626:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573644.931628:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573644.931631:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573644.931633:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573644.931635:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573644.931638:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546330:546330:256:4294967295] 192.168.202.128@tcp LPNI seq info [546253:546253:8:4294967295] 00000400:00000200:0.0:1788573644.931644:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573644.931650:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573644.931654:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573644.931657:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573644.931661:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573644.931664:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573644.931680:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573644.931682:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573644.931685:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573644.931689:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573644.931693:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573644.931694:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573644.931697:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573644.931701:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516598784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931720:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573644.931724:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516598784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573669 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931730:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.931732:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573644.931736:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573644.931739:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573644.931742:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573644.931744:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309004969 00000100:00000001:0.0:1788573644.931747:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309008622 00000100:00000001:0.0:1788573644.931749:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309004969 00000100:00000001:0.0:1788573644.931751:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309008622 00000100:00000001:0.0:1788573644.931753:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573644.931755:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.931757:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573644.931761:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573644/real 1788573644] req@ffff8fc4ac822a00 x1875440516598784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573644.931769:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573644.931770:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573644.931772:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573644.931774:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573644.931776:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573644.931779:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573644.931783:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516598784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573644.931790:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516598784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931797:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573644.931804:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931824:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573644.931826:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573644.931831:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516598784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931839:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573644.931842:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516598784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931849:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573644.931852:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931854:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573644.931856:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931858:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573644.931860:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573644.931862:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573644.931866:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573644.931868:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573644.931872:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516598784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573644.931880:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516598784:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573644.931884:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931887:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573644.931889:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516598400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931896:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573644.931898:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573644.931900:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573644.931903:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573644.931904:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573644.931912:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573644.931914:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573644.931916:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788573644.931919:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573644.931920:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573644.931922:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788573644.931925:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573644.931926:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931928:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573644.931932:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516598528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931938:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573644.931940:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573644.931941:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573644.931943:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573644.931945:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573644.931946:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573644.931948:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573644.931950:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788573644.931952:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573644.931953:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573644.931955:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788573644.931957:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573644.931958:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931960:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573644.931964:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516598656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.931971:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573644.931972:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573644.931973:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573644.931975:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573644.931978:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573644.931979:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573644.931981:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573644.931982:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788573644.931985:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573644.931991:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573644.931993:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788573644.931995:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573644.931997:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.931999:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573644.932003:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516598784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573669 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573644.932010:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573644.932011:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573644.932013:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573644.932015:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573644.932017:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573644.932018:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573644.932020:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573644.932022:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788573644.932024:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573644.932025:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573644.932027:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788573644.932029:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573644.932030:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.932032:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573644.932035:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573644.932037:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573644.932039:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573644.932040:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573645.440976:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573645.440980:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573645.440981:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573645.440984:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573645.440987:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573645.440991:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573645.440995:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573645.440997:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573645.440999:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573646.464888:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573646.464893:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573646.464897:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573646.464900:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573646.464921:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573646.464925:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573646.464929:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573646.464932:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573646.464937:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573648.512889:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573648.512894:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573648.512897:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573648.512903:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573648.512908:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573648.512913:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573648.512921:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573648.512926:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573648.512931:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573648.512964:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573648.512969:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573648.512973:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573648.512976:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573648.512979:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573648.512982:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573648.512985:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573648.512987:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573648.512990:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573648.768886:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309008622, at 4309008640 00000100:00000010:0.0:1788573648.768902:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573649.216972:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573649.216977:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573649.216981:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573649.536911:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573649.536916:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573649.536919:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573649.536923:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573649.536929:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573649.536935:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573649.536963:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573649.536967:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573649.536974:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573650.048949:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788573650.048967:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788573650.048980:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1788573650.049157:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788573650.049163:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573650.049179:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573650.049187:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573650.049191:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.049279:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573650.049288:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573650.049294:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573650.049312:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6300 x1875440516598912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.049345:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573650.049352:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573650.049364:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573650.049371:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573650.049386:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573650.049403:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516598912:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573650.049419:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573650.049429:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573650.049437:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573650.049445:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573650.049455:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573650.049463:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573650.049475:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573650.049487:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573650.049500:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573650.049513:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516598912, portal 25 00000100:00000001:0.0:1788573650.049521:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573650.049526:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605329664 : -123400104221952 : ffff8fc4ac9f6300) 00000100:00000040:0.0:1788573650.049543:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6300 x1875440516598912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.049601:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573650.049610:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573650.049619:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573650.049627:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516598912, offset 0 00000400:00000200:0.0:1788573650.049642:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573650.049662:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573650.049674:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573650.049683:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573650.049694:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573650.049701:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573650.049706:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573650.049716:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546331:546331:256:4294967295] 192.168.202.128@tcp LPNI seq info [546254:546254:8:4294967295] 00000400:00000200:0.0:1788573650.049732:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573650.049750:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573650.049758:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573650.049768:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573650.049779:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573650.049787:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788573650.049840:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573650.049842:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573650.049846:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573650.049851:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573650.049857:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573650.049861:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573650.049867:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573650.049874:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6300 x1875440516598912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.049911:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573650.049920:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6300 x1875440516598912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050009:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.050025:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573650.050037:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573650.050051:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573650.050059:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309013742 00000100:00000010:0.0:1788573650.050069:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573650.050079:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573650.050103:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309013742 00000100:00000010:0.0:1788573650.050110:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b504e0. 00000100:00000001:0.0:1788573650.050117:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573650.050125:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573650.050133:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.050146:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573650.050165:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573650/real 1788573650] req@ffff8fc4ac9f6300 x1875440516598912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573650.050212:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573650.050217:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573650.050225:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573650.050237:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573650.050244:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573650.050253:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573650.050265:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6300 x1875440516598912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573650.050302:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6300 x1875440516598912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050331:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573650.050334:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.050342:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573650.050345:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573650.050358:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6300 x1875440516598912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050393:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573650.050404:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6300 x1875440516598912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050427:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573650.050434:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.050438:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573650.050440:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.050447:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573650.050450:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573650.050461:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573650.050476:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573650.050483:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573650.050495:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6300 x1875440516598912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573650.050564:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516598912:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573650.050574:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.050577:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573650.050583:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6300 x1875440516598912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050588:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573650.050591:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573650.050594:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573650.050600:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573650.050602:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573650.050603:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573650.050606:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573650.050609:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36c00. 02000000:00000001:0.0:1788573650.050613:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573650.050622:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573650.050624:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6300. 00000100:00000001:0.0:1788573650.050626:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573650.050628:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.050629:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.050636:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573650.050638:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.050640:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573650.050641:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573650.050642:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573650.050646:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4380 x1875440516599040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050652:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573650.050653:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573650.050655:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573650.050657:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573650.050659:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573650.050661:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599040:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573650.050663:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573650.050664:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573650.050666:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573650.050667:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573650.050669:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573650.050671:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573650.050674:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573650.050676:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573650.050679:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573650.050682:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516599040, portal 10 00000100:00000001:0.0:1788573650.050683:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573650.050685:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605321600 : -123400104230016 : ffff8fc4ac9f4380) 00000100:00000040:0.0:1788573650.050690:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4380 x1875440516599040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050695:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573650.050697:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573650.050702:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573650.050704:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516599040, offset 0 00000400:00000200:0.0:1788573650.050707:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573650.050711:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573650.050714:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573650.050716:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573650.050718:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573650.050720:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573650.050722:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573650.050725:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546332:546332:256:4294967295] 192.168.202.128@tcp LPNI seq info [546255:546255:8:4294967295] 00000400:00000200:0.0:1788573650.050727:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573650.050732:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573650.050736:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573650.050738:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573650.050741:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573650.050743:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788573650.050752:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573650.050753:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573650.050755:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573650.050757:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573650.050760:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573650.050762:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573650.050765:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573650.050769:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4380 x1875440516599040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050778:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573650.050782:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4380 x1875440516599040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050790:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.050792:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573650.050845:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573650.050850:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573650.050854:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573650.050857:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309009940 00000100:00000001:0.0:1788573650.050860:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309013742 00000100:00000001:0.0:1788573650.050863:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309009940 00000100:00000001:0.0:1788573650.050865:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309013742 00000100:00000001:0.0:1788573650.050868:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573650.050870:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.050873:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573650.050878:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573650/real 1788573650] req@ffff8fc4ac9f4380 x1875440516599040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573650.050888:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573650.050890:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573650.050892:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573650.050895:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573650.050897:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573650.050899:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573650.050903:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4380 x1875440516599040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573650.050912:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4380 x1875440516599040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050920:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573650.050922:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.050924:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573650.050925:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573650.050929:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4380 x1875440516599040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050937:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573650.050942:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4380 x1875440516599040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050955:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573650.050959:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.050960:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573650.050961:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.050963:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573650.050965:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573650.050967:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573650.050969:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573650.050971:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573650.050974:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4380 x1875440516599040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573650.050980:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599040:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573650.050983:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573650.050985:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4a80 x1875440516599168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.050990:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573650.050991:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573650.050993:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573650.050994:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573650.050996:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573650.050998:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599168:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573650.051000:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573650.051001:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573650.051002:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573650.051004:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573650.051006:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573650.051007:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573650.051013:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573650.051015:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573650.051017:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573650.051020:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516599168, portal 10 00000100:00000001:0.0:1788573650.051021:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573650.051022:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605323392 : -123400104228224 : ffff8fc4ac9f4a80) 00000100:00000040:0.0:1788573650.051028:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4a80 x1875440516599168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051032:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573650.051034:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573650.051036:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573650.051037:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516599168, offset 0 00000400:00000200:0.0:1788573650.051040:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573650.051043:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573650.051045:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573650.051047:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573650.051049:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573650.051052:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573650.051053:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573650.051055:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546333:546333:256:4294967295] 192.168.202.128@tcp LPNI seq info [546256:546256:8:4294967295] 00000400:00000200:0.0:1788573650.051059:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573650.051064:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573650.051066:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573650.051068:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573650.051071:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573650.051073:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788573650.051081:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573650.051083:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573650.051100:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573650.051104:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573650.051107:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573650.051112:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573650.051115:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573650.051118:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4a80 x1875440516599168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051124:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573650.051126:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4a80 x1875440516599168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051131:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.051132:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573650.051134:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573650.051137:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573650.051140:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573650.051141:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309009947 00000100:00000001:0.0:1788573650.051143:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309013742 00000100:00000001:0.0:1788573650.051145:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309009947 00000100:00000001:0.0:1788573650.051146:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309013742 00000100:00000001:0.0:1788573650.051148:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573650.051149:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.051150:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573650.051152:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573650/real 1788573650] req@ffff8fc4ac9f4a80 x1875440516599168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573650.051158:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573650.051159:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573650.051160:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573650.051162:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573650.051163:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573650.051165:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573650.051167:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4a80 x1875440516599168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573650.051173:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4a80 x1875440516599168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051182:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573650.051183:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051184:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573650.051185:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573650.051188:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4a80 x1875440516599168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051192:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573650.051195:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4a80 x1875440516599168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051200:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573650.051201:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051202:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573650.051203:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051204:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573650.051206:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573650.051208:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573650.051210:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573650.051211:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573650.051214:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4a80 x1875440516599168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573650.051219:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599168:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573650.051222:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573650.051224:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7b80 x1875440516599296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051228:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573650.051229:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573650.051234:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573650.051236:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573650.051237:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573650.051239:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599296:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573650.051242:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573650.051244:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573650.051245:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573650.051246:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573650.051249:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573650.051250:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573650.051253:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573650.051254:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573650.051257:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573650.051260:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516599296, portal 4 00000100:00000001:0.0:1788573650.051261:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573650.051262:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335936 : -123400104215680 : ffff8fc4ac9f7b80) 00000100:00000040:0.0:1788573650.051267:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7b80 x1875440516599296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051271:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573650.051273:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573650.051274:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573650.051276:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516599296, offset 0 00000400:00000200:0.0:1788573650.051278:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573650.051281:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573650.051283:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573650.051285:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573650.051287:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573650.051289:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573650.051290:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573650.051293:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546334:546334:256:4294967295] 192.168.202.128@tcp LPNI seq info [546257:546257:8:4294967295] 00000400:00000200:0.0:1788573650.051296:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573650.051300:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573650.051307:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573650.051309:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573650.051312:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573650.051314:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788573650.051321:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573650.051323:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573650.051325:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573650.051327:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573650.051329:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573650.051330:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573650.051332:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573650.051335:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7b80 x1875440516599296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051340:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573650.051342:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7b80 x1875440516599296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051346:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.051347:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573650.051349:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573650.051351:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573650.051353:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573650.051354:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309010087 00000100:00000001:0.0:1788573650.051356:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309013742 00000100:00000001:0.0:1788573650.051358:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309010087 00000100:00000001:0.0:1788573650.051359:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309013742 00000100:00000001:0.0:1788573650.051361:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573650.051362:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.051363:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573650.051365:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573650/real 1788573650] req@ffff8fc4ac9f7b80 x1875440516599296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573650.051375:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573650.051376:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573650.051377:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573650.051379:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573650.051380:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573650.051381:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573650.051384:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7b80 x1875440516599296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573650.051389:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7b80 x1875440516599296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051395:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573650.051396:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051398:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573650.051399:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573650.051403:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7b80 x1875440516599296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051410:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573650.051415:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7b80 x1875440516599296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051422:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573650.051425:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051427:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573650.051429:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051431:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573650.051433:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573650.051436:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573650.051439:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573650.051442:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573650.051453:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7b80 x1875440516599296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573650.051463:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599296:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573650.051467:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573650.051472:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5880 x1875440516599424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051479:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573650.051481:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573650.051483:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573650.051485:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573650.051488:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573650.051491:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599424:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573650.051495:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573650.051497:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573650.051499:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573650.051501:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573650.051505:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573650.051507:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573650.051511:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573650.051514:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573650.051518:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573650.051521:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516599424, portal 4 00000100:00000001:0.0:1788573650.051523:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573650.051525:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605326976 : -123400104224640 : ffff8fc4ac9f5880) 00000100:00000040:0.0:1788573650.051532:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5880 x1875440516599424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051539:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573650.051542:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573650.051544:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573650.051546:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516599424, offset 0 00000400:00000200:0.0:1788573650.051554:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573650.051558:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573650.051562:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573650.051565:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573650.051568:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573650.051571:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573650.051573:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573650.051576:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546335:546335:256:4294967295] 192.168.202.128@tcp LPNI seq info [546258:546258:8:4294967295] 00000400:00000200:0.0:1788573650.051582:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573650.051589:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573650.051594:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573650.051598:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573650.051602:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573650.051605:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788573650.051617:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573650.051619:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573650.051622:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573650.051626:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573650.051629:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573650.051630:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573650.051632:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573650.051635:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5880 x1875440516599424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051641:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573650.051643:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5880 x1875440516599424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573675 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051648:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.051649:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573650.051651:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573650.051653:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573650.051655:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573650.051661:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309010089 00000100:00000001:0.0:1788573650.051663:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309013742 00000100:00000001:0.0:1788573650.051665:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309010089 00000100:00000001:0.0:1788573650.051667:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309013742 00000100:00000001:0.0:1788573650.051668:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573650.051669:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.051671:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573650.051673:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573650/real 1788573650] req@ffff8fc4ac9f5880 x1875440516599424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573650.051678:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573650.051679:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573650.051680:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573650.051682:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573650.051683:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573650.051685:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573650.051687:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5880 x1875440516599424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573650.051693:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5880 x1875440516599424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051697:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573650.051698:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051699:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573650.051700:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573650.051702:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5880 x1875440516599424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051707:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573650.051711:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5880 x1875440516599424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051718:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573650.051728:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051730:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573650.051731:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051733:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573650.051736:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573650.051739:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573650.051743:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573650.051746:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573650.051751:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5880 x1875440516599424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573650.051760:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599424:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573650.051764:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051767:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573650.051770:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4380 x1875440516599040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051778:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573650.051779:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573650.051782:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573650.051785:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573650.051786:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573650.051788:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573650.051790:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573650.051793:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34400. 02000000:00000001:0.0:1788573650.051796:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573650.051797:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573650.051799:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4380. 00000100:00000001:0.0:1788573650.051802:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573650.051804:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051805:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573650.051828:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4a80 x1875440516599168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051849:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573650.051851:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573650.051853:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573650.051856:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573650.051857:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573650.051859:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573650.051861:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573650.051863:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788573650.051865:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573650.051867:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573650.051868:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4a80. 00000100:00000001:0.0:1788573650.051871:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573650.051873:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051875:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573650.051879:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7b80 x1875440516599296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051887:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573650.051888:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573650.051890:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573650.051892:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573650.051893:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573650.051894:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573650.051896:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573650.051897:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788573650.051900:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573650.051901:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573650.051902:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7b80. 00000100:00000001:0.0:1788573650.051905:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573650.051906:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051907:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573650.051910:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5880 x1875440516599424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573675 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573650.051914:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573650.051915:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573650.051920:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573650.051922:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573650.051923:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573650.051924:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573650.051925:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573650.051926:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788573650.051928:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573650.051928:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573650.051929:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5880. 00000100:00000001:0.0:1788573650.051930:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573650.051931:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051933:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.051935:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573650.051937:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.051938:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573650.051939:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573650.560959:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573650.560971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573650.560977:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573650.560989:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573650.560998:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573650.561004:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.561040:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573650.561044:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573650.561048:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573651.584953:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573651.584963:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573651.584967:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573651.584985:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573651.584989:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573651.584995:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573651.585003:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573651.585009:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573651.585013:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573651.585111:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573651.585117:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573651.585121:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573651.585126:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573651.585141:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573651.585146:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573651.585151:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573651.585155:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573651.585158:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573652.609066:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573652.609070:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573652.609071:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573652.609075:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573652.609078:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573652.609081:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573652.609100:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573652.609102:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573652.609104:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573653.632950:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573653.632957:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573653.632961:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573653.632968:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573653.632973:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573653.632976:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573653.632981:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573653.632984:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573653.632988:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573653.888899:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309013742, at 4309013760 00000100:00000010:0.0:1788573653.888916:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573654.337024:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573654.337027:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573654.337030:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573654.656913:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573654.656922:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573654.656927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573654.656936:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573654.656944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573654.656950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573654.657003:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573654.657010:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573654.657016:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.169171:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573655.169175:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573655.169180:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573655.169188:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516599552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.169202:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573655.169204:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573655.169207:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573655.169210:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573655.169214:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573655.169221:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516599552:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573655.169234:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573655.169239:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573655.169242:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573655.169245:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573655.169251:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573655.169255:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573655.169263:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573655.169269:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573655.169277:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573655.169286:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516599552, portal 25 00000100:00000001:0.0:1788573655.169289:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573655.169292:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788573655.169301:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516599552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.169312:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573655.169317:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573655.169322:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573655.169327:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516599552, offset 0 00000400:00000200:0.0:1788573655.169335:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573655.169342:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573655.169349:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573655.169356:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573655.169371:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573655.169378:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573655.169386:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573655.169395:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546336:546336:256:4294967295] 192.168.202.128@tcp LPNI seq info [546259:546259:8:4294967295] 00000400:00000200:0.0:1788573655.169408:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573655.169428:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573655.169443:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573655.169452:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573655.169466:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573655.169474:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788573655.169512:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573655.169519:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573655.169529:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573655.169539:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573655.169549:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573655.169556:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573655.169566:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573655.169574:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516599552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.169596:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573655.169605:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516599552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.169622:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.169627:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573655.169636:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573655.169645:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573655.169651:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309018861 00000100:00000010:0.0:1788573655.169658:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573655.169667:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573655.169677:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309018861 00000100:00000010:0.0:1788573655.169692:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b500c0. 00000100:00000001:0.0:1788573655.169699:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573655.169707:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573655.169713:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.169718:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573655.169727:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573655/real 1788573655] req@ffff8fc4ac823480 x1875440516599552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573655.169748:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573655.169752:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573655.169755:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573655.169758:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573655.169761:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573655.169766:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573655.169771:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516599552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573655.169787:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516599552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.169795:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573655.169799:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.169806:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573655.169825:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573655.169829:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516599552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.169835:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573655.169841:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516599552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.169852:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573655.169854:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.169859:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573655.169861:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.169870:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573655.169873:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573655.169878:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573655.169885:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573655.169889:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573655.169896:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516599552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573655.169909:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516599552:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573655.169915:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.169919:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573655.169925:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516599552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.169938:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573655.169942:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573655.169948:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573655.169954:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573655.169970:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573655.169975:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573655.169982:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573655.169987:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788573655.169993:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573655.169998:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573655.170002:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788573655.170008:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573655.170012:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170016:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170024:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573655.170027:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.170031:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573655.170032:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573655.170033:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573655.170038:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516599680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170072:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573655.170075:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573655.170077:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573655.170081:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573655.170085:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573655.170090:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599680:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573655.170097:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573655.170100:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573655.170101:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573655.170105:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573655.170111:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573655.170114:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573655.170120:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573655.170123:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573655.170128:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573655.170133:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516599680, portal 10 00000100:00000001:0.0:1788573655.170138:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573655.170140:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788573655.170149:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516599680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170165:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573655.170171:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573655.170175:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573655.170179:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516599680, offset 0 00000400:00000200:0.0:1788573655.170186:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573655.170191:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573655.170195:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573655.170200:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573655.170205:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573655.170208:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573655.170210:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573655.170218:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546337:546337:256:4294967295] 192.168.202.128@tcp LPNI seq info [546260:546260:8:4294967295] 00000400:00000200:0.0:1788573655.170222:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573655.170230:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573655.170233:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573655.170236:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573655.170239:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573655.170241:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788573655.170249:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573655.170251:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573655.170253:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573655.170255:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573655.170257:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573655.170259:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573655.170261:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573655.170263:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516599680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170269:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573655.170271:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516599680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170276:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.170277:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573655.170279:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573655.170282:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573655.170284:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573655.170285:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309015059 00000100:00000001:0.0:1788573655.170287:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309018861 00000100:00000001:0.0:1788573655.170289:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309015059 00000100:00000001:0.0:1788573655.170290:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309018861 00000100:00000001:0.0:1788573655.170292:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573655.170296:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.170298:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573655.170301:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573655/real 1788573655] req@ffff8fc4ac822d80 x1875440516599680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573655.170306:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573655.170307:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573655.170309:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573655.170310:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573655.170312:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573655.170313:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573655.170315:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516599680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573655.170320:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516599680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170324:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573655.170325:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170326:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573655.170327:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573655.170329:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516599680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170333:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573655.170336:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516599680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170340:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573655.170342:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170343:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573655.170344:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170345:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573655.170347:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573655.170349:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573655.170354:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573655.170355:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573655.170358:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516599680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573655.170364:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599680:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573655.170366:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573655.170368:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516599808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170372:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573655.170373:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573655.170375:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573655.170376:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573655.170377:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573655.170379:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599808:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573655.170381:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573655.170382:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573655.170383:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573655.170384:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573655.170386:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573655.170387:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573655.170390:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573655.170391:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573655.170393:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573655.170396:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516599808, portal 10 00000100:00000001:0.0:1788573655.170397:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573655.170398:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788573655.170402:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516599808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170406:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573655.170411:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573655.170412:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573655.170414:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516599808, offset 0 00000400:00000200:0.0:1788573655.170417:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573655.170420:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573655.170422:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573655.170424:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573655.170426:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573655.170428:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573655.170429:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573655.170432:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546338:546338:256:4294967295] 192.168.202.128@tcp LPNI seq info [546261:546261:8:4294967295] 00000400:00000200:0.0:1788573655.170434:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573655.170439:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573655.170442:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573655.170444:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573655.170447:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573655.170448:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788573655.170455:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573655.170457:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573655.170459:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573655.170461:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573655.170463:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573655.170464:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573655.170466:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573655.170468:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516599808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170474:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573655.170476:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516599808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170480:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.170484:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573655.170487:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573655.170489:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573655.170491:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573655.170493:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309015066 00000100:00000001:0.0:1788573655.170494:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309018861 00000100:00000001:0.0:1788573655.170496:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309015066 00000100:00000001:0.0:1788573655.170497:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309018861 00000100:00000001:0.0:1788573655.170499:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573655.170500:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.170501:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573655.170504:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573655/real 1788573655] req@ffff8fc4ac821f80 x1875440516599808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573655.170508:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573655.170509:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573655.170510:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573655.170512:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573655.170513:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573655.170514:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573655.170516:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516599808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573655.170521:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516599808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170525:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573655.170526:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170527:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573655.170528:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573655.170530:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516599808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170534:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573655.170540:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516599808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170545:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573655.170547:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170548:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573655.170549:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170550:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573655.170552:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573655.170554:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573655.170557:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573655.170559:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573655.170563:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516599808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573655.170572:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599808:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573655.170576:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573655.170579:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516599936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170586:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573655.170588:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573655.170591:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573655.170593:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573655.170595:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573655.170598:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599936:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573655.170601:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573655.170602:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573655.170604:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573655.170605:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573655.170607:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573655.170609:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573655.170616:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573655.170618:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573655.170620:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573655.170623:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516599936, portal 4 00000100:00000001:0.0:1788573655.170624:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573655.170625:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788573655.170630:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516599936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170634:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573655.170636:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573655.170637:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573655.170639:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516599936, offset 0 00000400:00000200:0.0:1788573655.170641:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573655.170644:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573655.170646:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573655.170648:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573655.170650:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573655.170651:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573655.170653:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573655.170655:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546339:546339:256:4294967295] 192.168.202.128@tcp LPNI seq info [546262:546262:8:4294967295] 00000400:00000200:0.0:1788573655.170658:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573655.170662:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573655.170665:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573655.170668:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573655.170670:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573655.170672:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788573655.170680:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573655.170681:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573655.170683:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573655.170685:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573655.170687:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573655.170692:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573655.170694:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573655.170696:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516599936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170702:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573655.170704:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516599936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170708:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.170709:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573655.170712:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573655.170714:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573655.170716:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573655.170717:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309015206 00000100:00000001:0.0:1788573655.170719:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309018861 00000100:00000001:0.0:1788573655.170720:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309015206 00000100:00000001:0.0:1788573655.170722:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309018861 00000100:00000001:0.0:1788573655.170723:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573655.170724:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.170725:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573655.170727:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573655/real 1788573655] req@ffff8fc4ac822300 x1875440516599936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573655.170732:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573655.170733:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573655.170734:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573655.170736:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573655.170737:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573655.170738:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573655.170740:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516599936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573655.170745:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516599936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170752:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573655.170754:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170755:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573655.170756:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573655.170759:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516599936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170763:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573655.170766:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516599936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170770:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573655.170771:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170772:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573655.170773:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170774:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573655.170775:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573655.170777:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573655.170779:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573655.170780:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573655.170782:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516599936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573655.170787:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516599936:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573655.170789:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170791:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573655.170793:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516599680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170797:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573655.170798:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573655.170802:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573655.170803:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573655.170804:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573655.170806:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573655.170823:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573655.170824:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788573655.170826:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573655.170827:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573655.170829:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788573655.170831:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573655.170832:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170833:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573655.170836:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516599808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170841:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573655.170841:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573655.170843:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573655.170844:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573655.170845:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573655.170845:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573655.170847:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573655.170847:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788573655.170849:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573655.170849:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573655.170850:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788573655.170851:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573655.170852:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170853:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573655.170855:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516599936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170859:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573655.170859:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573655.170860:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573655.170861:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573655.170862:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573655.170863:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573655.170868:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573655.170869:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788573655.170870:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573655.170871:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573655.170872:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788573655.170873:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573655.170874:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170875:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.170878:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573655.170879:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.170880:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573655.170881:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573655.170882:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573655.170884:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516600064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170889:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573655.170890:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573655.170891:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573655.170893:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573655.170894:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573655.170896:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516600064:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573655.170898:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573655.170899:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573655.170900:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573655.170901:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573655.170903:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573655.170904:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573655.170907:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573655.170908:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573655.170911:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573655.170913:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516600064, portal 4 00000100:00000001:0.0:1788573655.170914:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573655.170915:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788573655.170918:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516600064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.170930:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573655.170931:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573655.170933:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573655.170935:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516600064, offset 0 00000400:00000200:0.0:1788573655.170937:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573655.170940:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573655.170943:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573655.170945:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573655.170948:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573655.170950:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573655.170952:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573655.170955:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546340:546340:256:4294967295] 192.168.202.128@tcp LPNI seq info [546263:546263:8:4294967295] 00000400:00000200:0.0:1788573655.170970:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573655.170977:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573655.170982:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573655.170986:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573655.170990:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573655.170993:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788573655.171005:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573655.171007:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573655.171010:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573655.171013:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573655.171015:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573655.171017:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573655.171019:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573655.171021:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516600064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.171027:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573655.171029:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516600064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573680 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.171043:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.171044:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573655.171047:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573655.171049:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573655.171052:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573655.171053:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309015209 00000100:00000001:0.0:1788573655.171055:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309018861 00000100:00000001:0.0:1788573655.171056:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309015209 00000100:00000001:0.0:1788573655.171058:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309018861 00000100:00000001:0.0:1788573655.171059:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573655.171060:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.171061:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573655.171064:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573655/real 1788573655] req@ffff8fc4ac820700 x1875440516600064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573655.171069:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573655.171070:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573655.171071:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573655.171073:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573655.171074:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573655.171075:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573655.171077:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516600064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573655.171083:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516600064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.171087:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573655.171088:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.171089:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573655.171090:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573655.171092:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516600064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.171099:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573655.171102:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516600064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.171106:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573655.171108:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.171109:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573655.171110:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.171111:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573655.171112:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573655.171114:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573655.171116:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573655.171117:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573655.171120:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516600064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573655.171125:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516600064:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573655.171127:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.171128:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573655.171130:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516600064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573680 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573655.171134:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573655.171135:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573655.171136:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573655.171137:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573655.171138:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573655.171139:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573655.171140:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573655.171141:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788573655.171142:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573655.171146:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573655.171147:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788573655.171149:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573655.171150:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.171151:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.171154:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573655.171155:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.171156:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573655.171157:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.680901:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573655.680907:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573655.680912:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573655.680920:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573655.680925:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573655.680929:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.680932:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573655.680935:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.680940:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573655.681039:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573655.681044:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573655.681047:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573655.681051:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573655.681055:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573655.681059:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.681063:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573655.681064:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573655.681066:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573656.704996:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573656.705003:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573656.705007:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573656.705012:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573656.705017:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573656.705021:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573656.705025:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573656.705027:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573656.705030:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573656.705046:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573656.705048:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573656.705050:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573656.705065:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573656.705067:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573656.705068:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573656.705069:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573656.705071:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573656.705072:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573657.728928:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573657.728934:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573657.728937:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573657.728942:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573657.728947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573657.728951:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573657.728956:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573657.728958:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573657.728963:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573658.752937:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573658.752945:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573658.752948:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573658.752957:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573658.752964:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573658.752970:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573658.752978:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573658.752981:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573658.752986:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573659.008949:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309018861, at 4309018880 00000100:00000010:0.0:1788573659.008965:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573659.457014:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573659.457029:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573659.457031:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573659.776934:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573659.776944:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573659.776949:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573659.776959:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573659.776968:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573659.776975:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573659.776983:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573659.776988:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573659.777011:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573659.777052:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573659.777054:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573659.777056:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573659.777058:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573659.777060:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573659.777062:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573659.777065:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573659.777066:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573659.777068:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.289150:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573660.289154:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573660.289159:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573660.289168:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516600192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.289188:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573660.289192:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573660.289196:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573660.289200:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573660.289204:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573660.289210:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516600192:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573660.289214:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573660.289217:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573660.289220:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573660.289224:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573660.289231:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573660.289234:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573660.289241:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573660.289247:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573660.289255:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573660.289262:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516600192, portal 25 00000100:00000001:0.0:1788573660.289266:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573660.289268:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788573660.289277:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516600192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.289314:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573660.289321:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573660.289325:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573660.289330:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516600192, offset 0 00000400:00000200:0.0:1788573660.289337:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573660.289346:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573660.289355:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573660.289360:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573660.289366:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573660.289371:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573660.289377:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573660.289384:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546341:546341:256:4294967295] 192.168.202.128@tcp LPNI seq info [546264:546264:8:4294967295] 00000400:00000200:0.0:1788573660.289393:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573660.289416:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573660.289426:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573660.289433:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573660.289443:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573660.289449:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788573660.289471:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573660.289474:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573660.289481:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573660.289488:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573660.289493:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573660.289496:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573660.289503:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573660.289509:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516600192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.289522:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573660.289529:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516600192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.289562:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.289565:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573660.289572:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573660.289577:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573660.289579:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309023981 00000100:00000010:0.0:1788573660.289585:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573660.289596:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573660.289607:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309023981 00000100:00000010:0.0:1788573660.289615:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50540. 00000100:00000001:0.0:1788573660.289623:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573660.289630:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573660.289639:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.289645:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573660.289653:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573660/real 1788573660] req@ffff8fc4ac820e00 x1875440516600192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573660.289666:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573660.289670:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573660.289673:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573660.289679:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573660.289683:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573660.289687:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573660.289694:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516600192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573660.289711:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516600192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.289723:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573660.289726:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.289729:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573660.289732:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573660.289738:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516600192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.289758:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573660.289767:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516600192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.289787:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573660.289796:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.289802:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573660.289857:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.289864:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573660.289867:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573660.289874:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573660.289882:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573660.289888:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573660.289897:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516600192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573660.289917:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516600192:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573660.289924:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.289928:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573660.289933:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516600192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.289949:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573660.289953:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573660.289958:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573660.289962:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573660.289967:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573660.289971:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573660.289976:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573660.289980:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788573660.289984:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573660.289992:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573660.289996:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788573660.290001:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573660.290005:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.290009:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.290017:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573660.290020:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.290024:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573660.290027:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573660.290049:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573660.290055:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516600320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290061:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573660.290062:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573660.290064:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573660.290066:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573660.290068:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573660.290071:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516600320:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573660.290073:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573660.290075:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573660.290076:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573660.290078:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573660.290080:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573660.290081:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573660.290084:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573660.290086:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573660.290089:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573660.290091:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516600320, portal 10 00000100:00000001:0.0:1788573660.290092:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573660.290093:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788573660.290097:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516600320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290102:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573660.290104:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573660.290109:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573660.290110:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516600320, offset 0 00000400:00000200:0.0:1788573660.290113:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573660.290116:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573660.290119:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573660.290121:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573660.290123:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573660.290125:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573660.290126:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573660.290129:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546342:546342:256:4294967295] 192.168.202.128@tcp LPNI seq info [546265:546265:8:4294967295] 00000400:00000200:0.0:1788573660.290131:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573660.290136:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573660.290139:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573660.290141:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573660.290144:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573660.290146:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788573660.290159:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573660.290160:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573660.290162:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573660.290165:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573660.290167:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573660.290168:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573660.290170:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573660.290173:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516600320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290178:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573660.290180:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516600320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290185:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.290186:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573660.290191:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573660.290194:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573660.290196:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573660.290198:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309020179 00000100:00000001:0.0:1788573660.290200:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309023981 00000100:00000001:0.0:1788573660.290201:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309020179 00000100:00000001:0.0:1788573660.290203:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309023981 00000100:00000001:0.0:1788573660.290204:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573660.290205:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.290207:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573660.290209:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573660/real 1788573660] req@ffff8fc4ac821880 x1875440516600320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573660.290214:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573660.290215:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573660.290217:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573660.290218:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573660.290220:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573660.290221:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573660.290225:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516600320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573660.290234:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516600320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290240:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573660.290242:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.290244:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573660.290246:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573660.290250:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516600320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290256:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573660.290260:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516600320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290272:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573660.290275:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.290277:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573660.290278:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.290280:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573660.290282:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573660.290284:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573660.290288:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573660.290291:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573660.290295:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516600320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573660.290303:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516600320:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573660.290307:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573660.290310:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516600448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290316:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573660.290317:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573660.290320:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573660.290322:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573660.290323:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573660.290326:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516600448:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573660.290329:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573660.290331:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573660.290332:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573660.290334:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573660.290337:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573660.290338:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573660.290347:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573660.290349:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573660.290352:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573660.290355:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516600448, portal 10 00000100:00000001:0.0:1788573660.290357:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573660.290358:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788573660.290364:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516600448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290370:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573660.290373:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573660.290375:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573660.290377:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516600448, offset 0 00000400:00000200:0.0:1788573660.290381:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573660.290385:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573660.290388:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573660.290391:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573660.290393:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573660.290396:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573660.290398:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573660.290402:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546343:546343:256:4294967295] 192.168.202.128@tcp LPNI seq info [546266:546266:8:4294967295] 00000400:00000200:0.0:1788573660.290407:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573660.290413:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573660.290418:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573660.290421:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573660.290424:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573660.290426:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788573660.290433:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573660.290434:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573660.290436:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573660.290439:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573660.290441:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573660.290446:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573660.290449:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573660.290452:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516600448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290457:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573660.290459:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516600448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290464:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.290465:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573660.290467:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573660.290469:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573660.290472:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573660.290473:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309020186 00000100:00000001:0.0:1788573660.290475:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309023981 00000100:00000001:0.0:1788573660.290476:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309020186 00000100:00000001:0.0:1788573660.290478:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309023981 00000100:00000001:0.0:1788573660.290479:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573660.290480:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.290481:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573660.290484:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573660/real 1788573660] req@ffff8fc4ac820a80 x1875440516600448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573660.290489:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573660.290490:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573660.290491:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573660.290493:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573660.290494:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573660.290495:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573660.290497:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516600448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573660.290502:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516600448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290510:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573660.290511:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.290512:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573660.290513:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573660.290516:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516600448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290520:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573660.290523:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516600448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290527:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573660.290529:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.290530:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573660.290530:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.290532:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573660.290533:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573660.290535:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573660.290537:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573660.290538:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573660.290540:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516600448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573660.290546:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516600448:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573660.290548:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573660.290550:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516600576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290554:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573660.290555:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573660.290559:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573660.290560:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573660.290562:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573660.290564:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516600576:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573660.290566:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573660.290567:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573660.290568:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573660.290569:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573660.290572:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573660.290573:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573660.290575:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573660.290577:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573660.290580:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573660.290582:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516600576, portal 4 00000100:00000001:0.0:1788573660.290584:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573660.290585:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788573660.290591:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516600576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290597:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573660.290599:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573660.290601:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573660.290604:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516600576, offset 0 00000400:00000200:0.0:1788573660.290608:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573660.290613:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573660.290617:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573660.290619:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573660.290622:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573660.290625:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573660.290628:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573660.290631:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546344:546344:256:4294967295] 192.168.202.128@tcp LPNI seq info [546267:546267:8:4294967295] 00000400:00000200:0.0:1788573660.290635:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573660.290642:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573660.290651:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573660.290653:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573660.290656:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573660.290658:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788573660.290665:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573660.290666:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573660.290668:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573660.290671:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573660.290673:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573660.290674:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573660.290676:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573660.290679:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516600576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290684:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573660.290686:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516600576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290691:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.290692:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573660.290694:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573660.290696:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573660.290698:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573660.290700:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309020326 00000100:00000001:0.0:1788573660.290702:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309023981 00000100:00000001:0.0:1788573660.290703:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309020326 00000100:00000001:0.0:1788573660.290705:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309023981 00000100:00000001:0.0:1788573660.290706:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573660.290707:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.290708:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573660.290711:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573660/real 1788573660] req@ffff8fc4ac823800 x1875440516600576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573660.290719:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573660.290720:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573660.290722:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573660.290723:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573660.290724:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573660.290726:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573660.290728:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516600576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573660.290734:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516600576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290738:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573660.290739:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.290740:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573660.290741:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573660.290743:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516600576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290747:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573660.290749:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516600576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290753:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573660.290755:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.290756:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573660.290756:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.290758:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573660.290759:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573660.290760:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573660.290763:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573660.290764:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573660.290770:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516600576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573660.290775:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516600576:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573660.290777:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573660.290779:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516600704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290783:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573660.290784:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573660.290786:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573660.290788:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573660.290790:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573660.290793:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516600704:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573660.290796:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573660.290798:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573660.290799:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573660.290800:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573660.290803:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573660.290806:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573660.290828:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573660.290844:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573660.290848:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573660.290850:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516600704, portal 4 00000100:00000001:0.0:1788573660.290852:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573660.290853:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788573660.290857:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516600704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290862:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573660.290864:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573660.290865:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573660.290867:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516600704, offset 0 00000400:00000200:0.0:1788573660.290874:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573660.290877:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573660.290879:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573660.290881:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573660.290883:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573660.290885:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573660.290886:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573660.290888:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546345:546345:256:4294967295] 192.168.202.128@tcp LPNI seq info [546268:546268:8:4294967295] 00000400:00000200:0.0:1788573660.290891:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573660.290895:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573660.290898:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573660.290900:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573660.290902:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573660.290904:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788573660.290911:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573660.290912:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573660.290915:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573660.290917:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573660.290919:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573660.290920:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573660.290922:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573660.290924:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516600704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290930:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573660.290932:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516600704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573685 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290936:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.290937:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573660.290939:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573660.290941:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573660.290943:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573660.290948:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309020329 00000100:00000001:0.0:1788573660.290950:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309023981 00000100:00000001:0.0:1788573660.290952:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309020329 00000100:00000001:0.0:1788573660.290953:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309023981 00000100:00000001:0.0:1788573660.290954:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573660.290956:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.290957:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573660.290960:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573660/real 1788573660] req@ffff8fc4ac821c00 x1875440516600704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573660.290965:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573660.290966:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573660.290967:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573660.290969:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573660.290970:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573660.290971:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573660.290973:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516600704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573660.290979:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516600704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290982:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573660.290983:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.290985:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573660.290985:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573660.290988:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516600704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290992:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573660.290994:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516600704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.290999:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573660.291003:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.291004:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573660.291005:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.291007:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573660.291008:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573660.291010:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573660.291012:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573660.291013:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573660.291016:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516600704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573660.291021:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516600704:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573660.291023:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.291024:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573660.291026:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516600320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.291030:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573660.291031:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573660.291032:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573660.291034:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573660.291035:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573660.291036:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573660.291037:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573660.291039:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788573660.291041:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573660.291041:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573660.291043:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788573660.291044:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573660.291045:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.291046:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573660.291048:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516600448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.291059:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573660.291060:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573660.291062:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573660.291063:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573660.291064:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573660.291065:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573660.291066:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573660.291067:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788573660.291068:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573660.291069:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573660.291069:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788573660.291071:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573660.291071:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.291073:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573660.291075:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516600576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.291078:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573660.291079:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573660.291080:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573660.291081:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573660.291082:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573660.291082:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573660.291084:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573660.291084:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788573660.291085:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573660.291086:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573660.291087:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788573660.291088:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573660.291089:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.291090:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573660.291092:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516600704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573685 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573660.291096:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573660.291097:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573660.291102:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573660.291103:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573660.291104:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573660.291105:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573660.291107:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573660.291108:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788573660.291109:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573660.291110:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573660.291110:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788573660.291112:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573660.291112:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.291113:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.291116:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573660.291117:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.291118:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573660.291119:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.800935:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573660.800945:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573660.800949:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573660.800966:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573660.800972:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573660.800977:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.800986:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573660.800990:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.800993:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573660.801010:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573660.801013:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573660.801015:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573660.801017:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573660.801019:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573660.801022:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.801024:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573660.801025:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573660.801028:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573661.824907:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573661.824914:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573661.824918:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573661.824927:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573661.824948:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573661.824952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573661.824965:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573661.824969:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573661.824972:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573662.848911:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573662.848920:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573662.848923:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573662.848931:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573662.848939:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573662.848945:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573662.848949:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573662.848952:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573662.848954:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573662.848971:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573662.848975:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573662.848979:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573662.848984:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573662.848989:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573662.848993:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573662.848997:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573662.849000:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573662.849004:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573663.872934:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573663.872942:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573663.872958:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573663.872963:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573663.872967:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573663.872970:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573663.872973:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573663.872976:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573663.872979:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573663.873013:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573663.873016:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573663.873017:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573663.873020:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573663.873025:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573663.873028:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573663.873039:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573663.873042:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573663.873047:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573664.128929:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309023981, at 4309024000 00000100:00000010:0.0:1788573664.128949:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573664.576989:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573664.576992:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573664.576994:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573664.896937:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573664.896945:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573664.896950:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573664.896959:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573664.896965:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573664.896970:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573664.896974:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573664.896977:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573664.896980:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573664.896997:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573664.897001:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573664.897003:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573664.897005:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573664.897007:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573664.897010:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573664.897011:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573664.897013:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573664.897014:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.409220:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573665.409226:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573665.409231:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573665.409240:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516600832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.409265:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573665.409269:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573665.409276:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573665.409281:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573665.409287:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573665.409307:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516600832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573665.409315:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573665.409327:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573665.409332:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573665.409339:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573665.409352:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573665.409359:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573665.409370:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573665.409376:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573665.409387:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573665.409398:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516600832, portal 25 00000100:00000001:0.0:1788573665.409403:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573665.409405:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788573665.409415:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516600832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.409445:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573665.409452:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573665.409459:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573665.409465:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516600832, offset 0 00000400:00000200:0.0:1788573665.409474:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573665.409504:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573665.409515:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573665.409527:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573665.409537:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573665.409545:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573665.409552:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573665.409556:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546346:546346:256:4294967295] 192.168.202.128@tcp LPNI seq info [546269:546269:8:4294967295] 00000400:00000200:0.0:1788573665.409568:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573665.409590:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573665.409602:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573665.409612:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573665.409630:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573665.409636:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788573665.409664:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573665.409671:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573665.409681:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573665.409694:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573665.409746:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573665.409753:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573665.409762:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573665.409773:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516600832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.409796:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573665.409804:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516600832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.409862:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.409867:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573665.409876:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573665.409886:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573665.409893:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309029102 00000100:00000010:0.0:1788573665.409901:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573665.409912:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573665.409922:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309029102 00000100:00000010:0.0:1788573665.409930:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b501e0. 00000100:00000001:0.0:1788573665.409938:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573665.409945:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573665.409952:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.409957:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573665.409966:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573665/real 1788573665] req@ffff8fc4ac821c00 x1875440516600832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573665.409993:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573665.410005:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573665.410010:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573665.410015:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573665.410018:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573665.410022:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573665.410031:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516600832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573665.410054:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516600832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410062:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573665.410064:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.410068:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573665.410073:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573665.410083:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516600832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410102:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573665.410114:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516600832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410133:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573665.410142:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.410145:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573665.410147:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.410150:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573665.410153:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573665.410157:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573665.410165:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573665.410168:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573665.410175:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516600832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573665.410194:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516600832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573665.410208:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.410213:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573665.410220:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516600832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410237:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573665.410242:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573665.410247:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573665.410254:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573665.410259:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573665.410264:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573665.410268:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573665.410274:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788573665.410281:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573665.410285:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573665.410289:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788573665.410295:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573665.410298:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.410303:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.410314:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573665.410319:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.410325:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573665.410329:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573665.410331:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573665.410337:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516600960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410346:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573665.410348:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573665.410350:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573665.410353:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573665.410356:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573665.410360:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516600960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573665.410364:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573665.410367:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573665.410369:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573665.410378:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573665.410381:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573665.410383:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573665.410388:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573665.410391:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573665.410396:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573665.410400:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516600960, portal 10 00000100:00000001:0.0:1788573665.410402:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573665.410404:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788573665.410411:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516600960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410419:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573665.410422:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573665.410424:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573665.410428:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516600960, offset 0 00000400:00000200:0.0:1788573665.410433:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573665.410437:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573665.410441:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573665.410445:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573665.410448:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573665.410451:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573665.410453:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573665.410457:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546347:546347:256:4294967295] 192.168.202.128@tcp LPNI seq info [546270:546270:8:4294967295] 00000400:00000200:0.0:1788573665.410461:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573665.410468:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573665.410473:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573665.410477:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573665.410482:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573665.410485:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788573665.410497:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573665.410499:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573665.410508:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573665.410512:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573665.410516:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573665.410518:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573665.410521:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573665.410525:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516600960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410534:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573665.410537:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516600960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410544:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.410545:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573665.410550:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573665.410555:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573665.410558:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573665.410560:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309025299 00000100:00000001:0.0:1788573665.410564:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309029102 00000100:00000001:0.0:1788573665.410566:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309025299 00000100:00000001:0.0:1788573665.410568:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309029102 00000100:00000001:0.0:1788573665.410570:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573665.410572:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.410575:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573665.410579:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573665/real 1788573665] req@ffff8fc4ac823800 x1875440516600960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573665.410587:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573665.410589:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573665.410590:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573665.410593:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573665.410594:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573665.410597:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573665.410606:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516600960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573665.410614:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516600960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410620:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573665.410622:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.410624:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573665.410626:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573665.410629:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516600960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410635:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573665.410639:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516600960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410645:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573665.410648:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.410649:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573665.410650:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.410652:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573665.410654:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573665.410657:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573665.410660:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573665.410663:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573665.410667:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516600960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573665.410675:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516600960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573665.410680:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573665.410684:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516601088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410713:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573665.410715:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573665.410719:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573665.410721:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573665.410723:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573665.410726:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573665.410730:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573665.410732:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573665.410734:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573665.410736:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573665.410740:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573665.410742:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573665.410745:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573665.410748:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573665.410752:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573665.410755:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516601088, portal 10 00000100:00000001:0.0:1788573665.410757:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573665.410759:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788573665.410766:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516601088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410772:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573665.410775:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573665.410778:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573665.410780:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516601088, offset 0 00000400:00000200:0.0:1788573665.410784:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573665.410789:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573665.410838:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573665.410842:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573665.410846:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573665.410849:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573665.410853:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573665.410862:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546348:546348:256:4294967295] 192.168.202.128@tcp LPNI seq info [546271:546271:8:4294967295] 00000400:00000200:0.0:1788573665.410867:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573665.410876:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573665.410883:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573665.410887:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573665.410892:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573665.410895:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788573665.410908:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573665.410910:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573665.410913:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573665.410917:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573665.410920:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573665.410922:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573665.410925:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573665.410929:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516601088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410938:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573665.410941:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516601088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.410957:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.410959:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573665.410963:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573665.410966:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573665.410970:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573665.410972:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309025307 00000100:00000001:0.0:1788573665.410975:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309029102 00000100:00000001:0.0:1788573665.410977:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309025307 00000100:00000001:0.0:1788573665.410980:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309029102 00000100:00000001:0.0:1788573665.410982:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573665.410991:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.410993:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573665.410998:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573665/real 1788573665] req@ffff8fc4ac820a80 x1875440516601088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573665.411007:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573665.411008:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573665.411011:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573665.411013:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573665.411015:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573665.411018:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573665.411022:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516601088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573665.411031:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516601088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411039:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573665.411041:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411043:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573665.411045:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573665.411049:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516601088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411058:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573665.411065:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516601088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411071:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573665.411073:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411074:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573665.411075:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411076:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573665.411079:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573665.411080:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573665.411088:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573665.411090:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573665.411093:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516601088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573665.411101:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573665.411105:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573665.411109:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516601216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411116:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573665.411118:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573665.411120:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573665.411122:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573665.411124:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573665.411128:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573665.411132:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573665.411134:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573665.411136:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573665.411138:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573665.411142:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573665.411144:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573665.411147:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573665.411149:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573665.411152:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573665.411154:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516601216, portal 4 00000100:00000001:0.0:1788573665.411156:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573665.411157:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788573665.411161:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516601216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411167:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573665.411173:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573665.411175:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573665.411177:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516601216, offset 0 00000400:00000200:0.0:1788573665.411180:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573665.411184:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573665.411188:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573665.411191:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573665.411192:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573665.411195:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573665.411199:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573665.411203:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546349:546349:256:4294967295] 192.168.202.128@tcp LPNI seq info [546272:546272:8:4294967295] 00000400:00000200:0.0:1788573665.411208:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573665.411216:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573665.411223:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573665.411227:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573665.411231:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573665.411234:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788573665.411246:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573665.411248:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573665.411251:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573665.411255:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573665.411259:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573665.411262:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573665.411265:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573665.411268:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516601216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411274:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573665.411276:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516601216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411280:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.411281:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573665.411289:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573665.411291:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573665.411294:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573665.411296:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309025447 00000100:00000001:0.0:1788573665.411299:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309029102 00000100:00000001:0.0:1788573665.411301:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309025447 00000100:00000001:0.0:1788573665.411303:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309029102 00000100:00000001:0.0:1788573665.411305:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573665.411308:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.411310:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573665.411315:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573665/real 1788573665] req@ffff8fc4ac821880 x1875440516601216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573665.411323:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573665.411324:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573665.411326:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573665.411329:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573665.411331:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573665.411333:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573665.411336:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516601216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573665.411344:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516601216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411350:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573665.411352:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411354:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573665.411355:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573665.411359:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516601216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411365:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573665.411369:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516601216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411382:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573665.411384:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411386:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573665.411388:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411390:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573665.411392:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573665.411395:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573665.411398:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573665.411401:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573665.411406:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516601216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573665.411414:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573665.411419:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573665.411422:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516601344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411429:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573665.411430:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573665.411433:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573665.411435:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573665.411437:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573665.411440:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573665.411443:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573665.411445:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573665.411447:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573665.411449:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573665.411452:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573665.411454:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573665.411458:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573665.411467:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573665.411471:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573665.411474:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516601344, portal 4 00000100:00000001:0.0:1788573665.411476:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573665.411478:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788573665.411485:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516601344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411492:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573665.411495:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573665.411498:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573665.411500:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516601344, offset 0 00000400:00000200:0.0:1788573665.411504:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573665.411508:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573665.411512:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573665.411515:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573665.411518:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573665.411520:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573665.411523:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573665.411526:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546350:546350:256:4294967295] 192.168.202.128@tcp LPNI seq info [546273:546273:8:4294967295] 00000400:00000200:0.0:1788573665.411530:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573665.411536:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573665.411541:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573665.411544:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573665.411549:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573665.411551:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788573665.411563:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573665.411565:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573665.411568:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573665.411572:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573665.411575:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573665.411577:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573665.411585:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573665.411588:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516601344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411594:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573665.411596:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516601344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573690 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411600:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.411601:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573665.411604:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573665.411606:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573665.411608:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573665.411610:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309025449 00000100:00000001:0.0:1788573665.411611:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309029102 00000100:00000001:0.0:1788573665.411613:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309025449 00000100:00000001:0.0:1788573665.411614:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309029102 00000100:00000001:0.0:1788573665.411616:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573665.411617:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.411619:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573665.411621:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573665/real 1788573665] req@ffff8fc4ac820e00 x1875440516601344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573665.411625:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573665.411626:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573665.411627:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573665.411629:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573665.411630:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573665.411631:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573665.411633:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516601344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573665.411638:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516601344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411649:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573665.411651:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411652:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573665.411653:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573665.411656:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516601344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411660:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573665.411663:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516601344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411667:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573665.411668:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411669:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573665.411670:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411671:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573665.411673:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573665.411674:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573665.411676:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573665.411677:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573665.411680:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516601344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573665.411684:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573665.411687:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411689:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573665.411691:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516600960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411695:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573665.411697:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573665.411701:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573665.411721:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573665.411724:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573665.411725:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573665.411728:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573665.411730:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788573665.411734:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573665.411736:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573665.411737:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788573665.411740:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573665.411742:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411744:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573665.411748:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516601088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411755:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573665.411756:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573665.411758:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573665.411761:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573665.411762:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573665.411763:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573665.411765:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573665.411766:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788573665.411768:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573665.411770:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573665.411771:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788573665.411773:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573665.411774:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411776:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573665.411779:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516601216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411785:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573665.411787:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573665.411788:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573665.411790:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573665.411791:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573665.411792:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573665.411800:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573665.411801:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788573665.411804:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573665.411805:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573665.411835:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788573665.411838:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573665.411839:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411841:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573665.411845:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516601344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573690 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573665.411853:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573665.411855:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573665.411857:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573665.411860:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573665.411862:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573665.411863:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573665.411866:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573665.411867:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788573665.411870:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573665.411871:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573665.411873:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788573665.411875:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573665.411876:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411879:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.411884:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573665.411886:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.411888:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573665.411890:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573665.920903:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573665.920910:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573665.920914:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573665.920920:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573665.920926:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573665.920932:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.920937:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573665.920942:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573665.920948:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573666.944899:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573666.944908:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573666.944912:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573666.944919:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573666.944926:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573666.944929:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573666.944933:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573666.944943:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573666.944947:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573666.944972:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573666.944978:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573666.944982:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573666.944986:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573666.944989:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573666.944992:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573666.944995:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573666.944997:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573666.944999:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573667.968923:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573667.968933:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573667.968935:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573667.968942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573667.968945:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573667.968951:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573667.968954:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573667.968958:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573667.968963:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573667.968978:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573667.968983:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573667.968987:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573667.968993:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573667.968996:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573667.968998:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573667.969001:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573667.969003:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573667.969005:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573668.992920:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573668.992930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573668.992954:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573668.992961:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573668.992968:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573668.992974:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573668.992993:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573668.992997:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573668.993001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573669.248903:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309029102, at 4309029120 00000100:00000010:0.0:1788573669.248925:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573669.697062:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573669.697066:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573669.697070:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573670.016916:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573670.016926:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573670.016931:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573670.016939:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573670.016953:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573670.016956:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.016976:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573670.016979:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.016984:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.529147:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573670.529153:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573670.529158:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573670.529182:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516601472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.529205:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573670.529208:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573670.529217:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573670.529221:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573670.529227:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573670.529236:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516601472:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573670.529241:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573670.529246:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573670.529249:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573670.529262:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573670.529268:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573670.529274:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573670.529285:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573670.529291:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573670.529300:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573670.529310:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516601472, portal 25 00000100:00000001:0.0:1788573670.529317:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573670.529321:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788573670.529343:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516601472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.529367:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573670.529376:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573670.529393:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573670.529401:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516601472, offset 0 00000400:00000200:0.0:1788573670.529417:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573670.529432:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573670.529445:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573670.529454:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573670.529463:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573670.529472:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573670.529479:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573670.529488:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546351:546351:256:4294967295] 192.168.202.128@tcp LPNI seq info [546274:546274:8:4294967295] 00000400:00000200:0.0:1788573670.529499:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573670.529528:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573670.529542:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573670.529552:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573670.529565:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573670.529584:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788573670.529619:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573670.529624:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573670.529638:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573670.529649:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573670.529655:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573670.529659:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573670.529667:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573670.529675:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516601472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.529692:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573670.529701:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516601472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.529717:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.529723:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573670.529731:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573670.529739:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573670.529743:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309034221 00000100:00000010:0.0:1788573670.529751:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573670.529761:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573670.529769:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309034221 00000100:00000010:0.0:1788573670.529775:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50840. 00000100:00000001:0.0:1788573670.529781:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573670.529786:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573670.529791:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.529795:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573670.529802:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573670/real 1788573670] req@ffff8fc4ac820700 x1875440516601472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573670.529849:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573670.529851:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573670.529853:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573670.529856:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573670.529857:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573670.529860:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573670.529864:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516601472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573670.529875:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516601472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.529880:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573670.529882:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.529883:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573670.529885:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573670.529888:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516601472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.529893:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573670.529897:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516601472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.529901:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573670.529903:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.529904:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573670.529905:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.529907:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573670.529908:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573670.529910:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573670.529912:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573670.529914:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573670.529916:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516601472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573670.529922:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516601472:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573670.529924:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.529926:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573670.529928:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516601472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.529939:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573670.529941:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573670.529943:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573670.529946:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573670.529948:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573670.529949:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573670.529950:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573670.529952:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788573670.529955:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573670.529957:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573670.529958:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788573670.529960:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573670.529961:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.529963:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.529968:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573670.529969:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.529971:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573670.529972:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573670.529973:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573670.529976:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516601600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.529988:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573670.529990:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573670.529991:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573670.529993:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573670.529994:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573670.529996:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601600:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573670.529999:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573670.530000:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573670.530001:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573670.530003:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573670.530005:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573670.530007:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573670.530009:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573670.530015:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573670.530018:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573670.530020:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516601600, portal 10 00000100:00000001:0.0:1788573670.530022:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573670.530023:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788573670.530027:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516601600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530032:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573670.530034:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573670.530035:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573670.530037:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516601600, offset 0 00000400:00000200:0.0:1788573670.530039:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573670.530042:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573670.530044:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573670.530047:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573670.530048:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573670.530050:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573670.530058:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573670.530060:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546352:546352:256:4294967295] 192.168.202.128@tcp LPNI seq info [546275:546275:8:4294967295] 00000400:00000200:0.0:1788573670.530063:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573670.530068:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573670.530072:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573670.530074:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573670.530077:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573670.530079:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788573670.530086:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573670.530087:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573670.530089:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573670.530092:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573670.530094:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573670.530095:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573670.530103:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573670.530105:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516601600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530111:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573670.530113:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516601600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530118:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.530119:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573670.530121:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573670.530124:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573670.530126:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573670.530127:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309030419 00000100:00000001:0.0:1788573670.530129:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309034221 00000100:00000001:0.0:1788573670.530131:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309030419 00000100:00000001:0.0:1788573670.530133:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309034221 00000100:00000001:0.0:1788573670.530134:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573670.530136:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.530137:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573670.530139:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573670/real 1788573670] req@ffff8fc4ac822300 x1875440516601600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573670.530144:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573670.530151:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573670.530152:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573670.530154:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573670.530155:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573670.530156:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573670.530158:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516601600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573670.530164:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516601600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530176:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573670.530177:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.530178:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573670.530179:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573670.530182:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516601600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530187:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573670.530189:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516601600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530193:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573670.530195:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.530196:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573670.530197:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.530198:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573670.530199:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573670.530201:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573670.530203:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573670.530204:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573670.530206:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516601600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573670.530211:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601600:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573670.530214:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573670.530216:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516601728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530220:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573670.530221:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573670.530226:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573670.530227:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573670.530229:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573670.530231:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601728:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573670.530233:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573670.530234:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573670.530236:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573670.530237:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573670.530239:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573670.530240:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573670.530242:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573670.530244:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573670.530246:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573670.530248:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516601728, portal 10 00000100:00000001:0.0:1788573670.530250:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573670.530250:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788573670.530254:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516601728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530259:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573670.530261:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573670.530262:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573670.530263:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516601728, offset 0 00000400:00000200:0.0:1788573670.530266:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573670.530268:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573670.530271:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573670.530273:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573670.530274:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573670.530276:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573670.530277:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573670.530279:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546353:546353:256:4294967295] 192.168.202.128@tcp LPNI seq info [546276:546276:8:4294967295] 00000400:00000200:0.0:1788573670.530282:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573670.530287:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573670.530293:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573670.530295:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573670.530298:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573670.530300:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788573670.530307:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573670.530308:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573670.530310:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573670.530312:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573670.530315:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573670.530316:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573670.530318:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573670.530320:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516601728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530325:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573670.530327:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516601728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530331:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.530332:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573670.530334:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573670.530336:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573670.530338:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573670.530339:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309030426 00000100:00000001:0.0:1788573670.530341:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309034221 00000100:00000001:0.0:1788573670.530342:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309030426 00000100:00000001:0.0:1788573670.530344:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309034221 00000100:00000001:0.0:1788573670.530345:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573670.530347:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.530348:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573670.530350:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573670/real 1788573670] req@ffff8fc4ac821f80 x1875440516601728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573670.530360:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573670.530360:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573670.530362:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573670.530363:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573670.530364:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573670.530366:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573670.530368:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516601728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573670.530373:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516601728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530377:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573670.530378:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.530379:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573670.530380:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573670.530382:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516601728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530386:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573670.530388:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516601728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530392:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573670.530394:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.530395:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573670.530396:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.530397:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573670.530399:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573670.530401:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573670.530402:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573670.530404:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573670.530409:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516601728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573670.530414:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601728:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573670.530417:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573670.530419:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516601856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530426:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573670.530427:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573670.530429:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573670.530431:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573670.530433:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573670.530436:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601856:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573670.530440:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573670.530442:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573670.530443:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573670.530445:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573670.530449:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573670.530451:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573670.530455:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573670.530458:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573670.530462:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573670.530465:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516601856, portal 4 00000100:00000001:0.0:1788573670.530467:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573670.530469:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788573670.530475:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516601856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530482:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573670.530484:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573670.530487:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573670.530489:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516601856, offset 0 00000400:00000200:0.0:1788573670.530501:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573670.530505:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573670.530509:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573670.530512:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573670.530515:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573670.530517:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573670.530520:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573670.530523:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546354:546354:256:4294967295] 192.168.202.128@tcp LPNI seq info [546277:546277:8:4294967295] 00000400:00000200:0.0:1788573670.530527:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573670.530534:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573670.530539:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573670.530543:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573670.530547:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573670.530550:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788573670.530569:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573670.530572:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573670.530639:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573670.530656:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573670.530661:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573670.530664:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573670.530668:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573670.530672:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516601856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530680:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573670.530682:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516601856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530687:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.530688:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573670.530692:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573670.530694:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573670.530697:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573670.530705:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309030566 00000100:00000001:0.0:1788573670.530707:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309034221 00000100:00000001:0.0:1788573670.530709:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309030566 00000100:00000001:0.0:1788573670.530710:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309034221 00000100:00000001:0.0:1788573670.530712:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573670.530714:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.530715:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573670.530718:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573670/real 1788573670] req@ffff8fc4ac822d80 x1875440516601856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573670.530724:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573670.530725:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573670.530726:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573670.530728:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573670.530729:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573670.530731:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573670.530733:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516601856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573670.530739:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516601856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530743:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573670.530743:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.530745:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573670.530746:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573670.530748:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516601856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530752:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573670.530755:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516601856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530760:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573670.530764:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.530766:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573670.530767:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.530768:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573670.530769:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573670.530771:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573670.530774:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573670.530775:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573670.530778:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516601856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573670.530783:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601856:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573670.530786:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573670.530788:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516601984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530793:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573670.530793:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573670.530795:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573670.530796:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573670.530798:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573670.530800:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601984:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573670.530802:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573670.530803:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573670.530804:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573670.530805:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573670.530826:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573670.530828:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573670.530830:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573670.530832:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573670.530835:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573670.530837:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516601984, portal 4 00000100:00000001:0.0:1788573670.530843:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573670.530844:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788573670.530849:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516601984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530853:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573670.530855:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573670.530857:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573670.530859:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516601984, offset 0 00000400:00000200:0.0:1788573670.530861:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573670.530864:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573670.530867:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573670.530870:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573670.530872:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573670.530874:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573670.530875:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573670.530877:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546355:546355:256:4294967295] 192.168.202.128@tcp LPNI seq info [546278:546278:8:4294967295] 00000400:00000200:0.0:1788573670.530880:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573670.530893:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573670.530897:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573670.530899:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573670.530902:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573670.530905:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788573670.530913:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573670.530914:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573670.530917:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573670.530919:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573670.530921:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573670.530922:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573670.530924:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573670.530927:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516601984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530940:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573670.530943:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516601984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573695 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530947:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.530949:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573670.530951:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573670.530953:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573670.530956:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573670.530957:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309030569 00000100:00000001:0.0:1788573670.530959:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309034221 00000100:00000001:0.0:1788573670.530961:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309030569 00000100:00000001:0.0:1788573670.530963:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309034221 00000100:00000001:0.0:1788573670.530964:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573670.530965:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.530966:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573670.530969:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573670/real 1788573670] req@ffff8fc4ac823480 x1875440516601984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573670.530974:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573670.530975:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573670.530976:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573670.530977:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573670.530978:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573670.530980:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573670.530982:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516601984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573670.530987:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516601984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.530991:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573670.530995:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.530996:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573670.530997:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573670.531000:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516601984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.531005:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573670.531007:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516601984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.531012:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573670.531013:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.531014:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573670.531015:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.531016:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573670.531017:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573670.531019:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573670.531021:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573670.531022:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573670.531024:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516601984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573670.531029:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516601984:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573670.531031:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.531033:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573670.531035:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516601600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.531040:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573670.531041:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573670.531043:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573670.531044:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573670.531045:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573670.531049:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573670.531051:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573670.531052:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788573670.531054:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573670.531055:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573670.531056:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788573670.531058:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573670.531059:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.531061:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573670.531063:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516601728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.531068:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573670.531068:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573670.531070:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573670.531071:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573670.531072:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573670.531072:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573670.531073:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573670.531074:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788573670.531075:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573670.531076:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573670.531077:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788573670.531078:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573670.531079:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.531080:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573670.531082:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516601856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.531086:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573670.531086:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573670.531087:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573670.531088:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573670.531089:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573670.531090:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573670.531091:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573670.531092:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788573670.531093:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573670.531097:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573670.531099:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788573670.531100:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573670.531101:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.531102:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573670.531105:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516601984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573695 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573670.531108:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573670.531109:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573670.531110:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573670.531112:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573670.531113:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573670.531114:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573670.531115:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573670.531116:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788573670.531117:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573670.531117:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573670.531118:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788573670.531119:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573670.531120:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.531121:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573670.531124:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573670.531125:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573670.531126:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573670.531127:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573671.040891:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573671.040899:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573671.040901:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573671.040909:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573671.040914:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573671.040918:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573671.040924:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573671.040929:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573671.040932:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573672.064919:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573672.064927:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573672.064931:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573672.064941:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573672.064967:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573672.064972:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573672.064988:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573672.064994:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573672.065002:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573673.088898:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573673.088902:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573673.088906:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573673.088914:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573673.088919:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573673.088923:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573673.088934:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573673.088937:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573673.088940:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573674.112878:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573674.112885:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573674.112889:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573674.112896:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573674.112902:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573674.112907:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573674.112912:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573674.112916:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573674.112921:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573674.112963:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573674.112966:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573674.112969:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573674.112975:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573674.112977:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573674.112979:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573674.112981:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573674.112982:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573674.112984:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573674.368894:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309034221, at 4309034240 00000100:00000010:0.0:1788573674.368904:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573674.817066:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573674.817071:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573674.817075:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573675.136893:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573675.136898:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573675.136903:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573675.136907:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573675.136911:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573675.136915:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.136944:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573675.136947:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.136950:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.649169:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573675.649175:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573675.649180:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573675.649190:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516602112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.649205:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573675.649208:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573675.649213:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573675.649217:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573675.649221:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573675.649227:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516602112:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573675.649235:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573675.649239:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573675.649243:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573675.649248:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573675.649255:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573675.649259:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573675.649267:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573675.649272:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573675.649279:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573675.649287:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516602112, portal 25 00000100:00000001:0.0:1788573675.649290:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573675.649294:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788573675.649299:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516602112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.649325:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573675.649330:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573675.649334:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573675.649338:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516602112, offset 0 00000400:00000200:0.0:1788573675.649346:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573675.649359:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573675.649367:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573675.649373:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573675.649379:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573675.649385:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573675.649389:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573675.649397:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546356:546356:256:4294967295] 192.168.202.128@tcp LPNI seq info [546279:546279:8:4294967295] 00000400:00000200:0.0:1788573675.649405:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573675.649420:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573675.649432:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573675.649438:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573675.649462:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573675.649465:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788573675.649503:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573675.649508:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573675.649514:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573675.649523:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573675.649530:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573675.649533:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573675.649540:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573675.649547:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516602112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.649564:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573675.649570:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516602112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.649599:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.649602:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573675.649610:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573675.649618:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573675.649623:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309039341 00000100:00000010:0.0:1788573675.649630:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573675.649655:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573675.649662:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309039341 00000100:00000010:0.0:1788573675.649666:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788573675.649669:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573675.649673:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573675.649677:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.649681:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573675.649689:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573675/real 1788573675] req@ffff8fc4ac822a00 x1875440516602112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573675.649721:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573675.649724:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573675.649729:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573675.649735:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573675.649739:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573675.649745:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573675.649752:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516602112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573675.649770:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516602112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.649788:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573675.649791:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.649794:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573675.649798:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573675.649807:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516602112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.649825:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573675.649831:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516602112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.649847:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573675.649903:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.649909:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573675.649912:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.649915:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573675.649918:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573675.649924:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573675.649931:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573675.649934:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573675.649946:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516602112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573675.649976:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516602112:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573675.649984:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.649987:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573675.649992:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516602112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650002:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573675.650009:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573675.650014:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573675.650019:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573675.650022:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573675.650024:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573675.650028:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573675.650032:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788573675.650036:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573675.650044:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573675.650047:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788573675.650058:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573675.650062:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.650083:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.650091:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573675.650095:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.650098:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573675.650103:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573675.650107:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573675.650113:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516602240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650128:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573675.650133:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573675.650155:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573675.650160:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573675.650169:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573675.650179:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516602240:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573675.650188:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573675.650193:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573675.650197:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573675.650204:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573675.650209:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573675.650215:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573675.650241:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573675.650249:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573675.650256:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573675.650262:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516602240, portal 10 00000100:00000001:0.0:1788573675.650266:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573675.650268:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788573675.650276:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516602240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650298:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573675.650301:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573675.650309:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573675.650311:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516602240, offset 0 00000400:00000200:0.0:1788573675.650314:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573675.650318:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573675.650321:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573675.650324:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573675.650326:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573675.650328:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573675.650330:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573675.650332:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546357:546357:256:4294967295] 192.168.202.128@tcp LPNI seq info [546280:546280:8:4294967295] 00000400:00000200:0.0:1788573675.650336:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573675.650342:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573675.650345:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573675.650348:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573675.650351:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573675.650353:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788573675.650362:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573675.650363:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573675.650366:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573675.650368:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573675.650371:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573675.650372:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573675.650375:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573675.650377:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516602240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650384:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573675.650386:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516602240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650391:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.650392:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573675.650399:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573675.650402:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573675.650405:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573675.650407:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309035539 00000100:00000001:0.0:1788573675.650409:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309039341 00000100:00000001:0.0:1788573675.650410:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309035539 00000100:00000001:0.0:1788573675.650412:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309039341 00000100:00000001:0.0:1788573675.650431:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573675.650432:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.650434:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573675.650437:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573675/real 1788573675] req@ffff8fc4ac823100 x1875440516602240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573675.650442:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573675.650443:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573675.650460:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573675.650462:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573675.650464:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573675.650465:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573675.650468:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516602240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573675.650475:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516602240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650480:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573675.650496:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.650498:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573675.650499:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573675.650501:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516602240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650505:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573675.650507:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516602240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650516:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573675.650518:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.650519:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573675.650520:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.650521:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573675.650523:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573675.650524:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573675.650527:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573675.650528:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573675.650531:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516602240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573675.650536:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516602240:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573675.650539:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573675.650540:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516602368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650544:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573675.650545:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573675.650547:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573675.650548:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573675.650549:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573675.650551:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516602368:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573675.650553:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573675.650555:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573675.650556:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573675.650557:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573675.650559:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573675.650560:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573675.650566:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573675.650568:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573675.650570:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573675.650572:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516602368, portal 10 00000100:00000001:0.0:1788573675.650574:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573675.650575:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788573675.650579:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516602368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650583:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573675.650585:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573675.650586:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573675.650588:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516602368, offset 0 00000400:00000200:0.0:1788573675.650590:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573675.650593:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573675.650595:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573675.650597:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573675.650599:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573675.650601:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573675.650602:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573675.650604:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546358:546358:256:4294967295] 192.168.202.128@tcp LPNI seq info [546281:546281:8:4294967295] 00000400:00000200:0.0:1788573675.650607:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573675.650612:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573675.650614:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573675.650616:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573675.650619:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573675.650621:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788573675.650628:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573675.650630:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573675.650632:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573675.650634:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573675.650636:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573675.650641:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573675.650643:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573675.650646:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516602368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650652:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573675.650654:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516602368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650658:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.650659:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573675.650661:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573675.650664:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573675.650666:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573675.650667:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309035546 00000100:00000001:0.0:1788573675.650669:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309039341 00000100:00000001:0.0:1788573675.650687:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309035546 00000100:00000001:0.0:1788573675.650689:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309039341 00000100:00000001:0.0:1788573675.650690:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573675.650691:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.650693:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573675.650695:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573675/real 1788573675] req@ffff8fc4ac821500 x1875440516602368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573675.650700:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573675.650701:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573675.650703:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573675.650705:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573675.650706:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573675.650707:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573675.650709:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516602368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573675.650715:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516602368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650723:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573675.650724:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.650726:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573675.650727:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573675.650730:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516602368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650734:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573675.650737:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516602368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650742:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573675.650744:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.650745:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573675.650745:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.650747:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573675.650748:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573675.650750:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573675.650752:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573675.650753:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573675.650756:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516602368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573675.650761:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516602368:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573675.650763:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573675.650766:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516602496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650771:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573675.650772:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573675.650776:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573675.650778:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573675.650780:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573675.650782:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516602496:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573675.650784:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573675.650786:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573675.650787:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573675.650788:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573675.650790:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573675.650792:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573675.650795:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573675.650796:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573675.650799:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573675.650801:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516602496, portal 4 00000100:00000001:0.0:1788573675.650802:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573675.650803:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788573675.650807:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516602496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650812:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573675.650814:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573675.650815:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573675.650816:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516602496, offset 0 00000400:00000200:0.0:1788573675.650819:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573675.650822:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573675.650824:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573675.650827:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573675.650828:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573675.650830:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573675.650832:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573675.650834:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546359:546359:256:4294967295] 192.168.202.128@tcp LPNI seq info [546282:546282:8:4294967295] 00000400:00000200:0.0:1788573675.650837:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573675.650858:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573675.650866:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573675.650868:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573675.650871:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573675.650889:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788573675.650896:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573675.650898:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573675.650900:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573675.650903:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573675.650905:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573675.650906:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573675.650908:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573675.650910:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516602496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650915:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573675.650917:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516602496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650922:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.650923:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573675.650925:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573675.650927:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573675.650929:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573675.650931:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309035687 00000100:00000001:0.0:1788573675.650932:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309039341 00000100:00000001:0.0:1788573675.650934:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309035687 00000100:00000001:0.0:1788573675.650935:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309039341 00000100:00000001:0.0:1788573675.650937:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573675.650938:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.650939:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573675.650941:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573675/real 1788573675] req@ffff8fc4ac823b80 x1875440516602496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573675.650950:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573675.650951:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573675.650953:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573675.650954:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573675.650955:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573675.650956:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573675.650959:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516602496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573675.650964:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516602496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650968:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573675.650969:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.650970:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573675.650971:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573675.650973:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516602496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650977:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573675.650979:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516602496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.650984:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573675.650985:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.650987:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573675.650988:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.650989:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573675.650990:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573675.650991:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573675.650993:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573675.650995:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573675.651001:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516602496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573675.651006:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516602496:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573675.651008:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.651010:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573675.651011:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516602240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.651015:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573675.651016:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573675.651018:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573675.651019:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573675.651020:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573675.651021:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573675.651022:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573675.651040:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788573675.651042:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573675.651044:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573675.651045:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788573675.651046:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573675.651047:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.651049:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573675.651051:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516602368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.651055:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573675.651056:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573675.651057:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573675.651059:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573675.651060:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573675.651060:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573675.651062:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573675.651062:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788573675.651064:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573675.651067:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573675.651069:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788573675.651070:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573675.651071:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.651073:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573675.651075:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516602496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.651079:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573675.651080:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573675.651081:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573675.651082:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573675.651083:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573675.651084:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573675.651085:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573675.651086:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788573675.651087:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573675.651088:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573675.651089:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788573675.651090:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573675.651091:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.651092:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.651095:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573675.651096:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.651097:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573675.651098:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573675.651099:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573675.651101:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516602624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.651106:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573675.651107:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573675.651108:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573675.651109:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573675.651111:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573675.651113:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516602624:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573675.651131:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573675.651139:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573675.651141:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573675.651143:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573675.651146:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573675.651148:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573675.651153:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573675.651155:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573675.651159:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573675.651163:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516602624, portal 4 00000100:00000001:0.0:1788573675.651165:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573675.651167:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788573675.651174:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516602624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.651182:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573675.651185:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573675.651187:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573675.651190:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516602624, offset 0 00000400:00000200:0.0:1788573675.651195:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573675.651199:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573675.651203:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573675.651206:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573675.651209:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573675.651212:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573675.651230:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573675.651233:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546360:546360:256:4294967295] 192.168.202.128@tcp LPNI seq info [546283:546283:8:4294967295] 00000400:00000200:0.0:1788573675.651238:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573675.651242:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573675.651245:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573675.651248:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573675.651251:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573675.651253:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788573675.651259:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573675.651265:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573675.651267:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573675.651270:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573675.651272:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573675.651274:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573675.651276:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573675.651278:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516602624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.651283:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573675.651286:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516602624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573700 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.651290:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.651291:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573675.651293:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573675.651295:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573675.651298:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573675.651299:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309035689 00000100:00000001:0.0:1788573675.651301:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309039341 00000100:00000001:0.0:1788573675.651302:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309035689 00000100:00000001:0.0:1788573675.651304:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309039341 00000100:00000001:0.0:1788573675.651305:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573675.651307:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.651308:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573675.651310:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573675/real 1788573675] req@ffff8fc4ac820000 x1875440516602624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573675.651315:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573675.651315:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573675.651317:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573675.651318:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573675.651319:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573675.651324:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573675.651326:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516602624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573675.651332:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516602624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.651336:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573675.651337:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.651338:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573675.651339:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573675.651341:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516602624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.651345:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573675.651348:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516602624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.651352:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573675.651353:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.651354:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573675.651355:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.651356:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573675.651358:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573675.651359:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573675.651361:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573675.651362:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573675.651364:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516602624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573675.651372:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516602624:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573675.651375:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.651377:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573675.651385:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516602624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573700 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573675.651391:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573675.651393:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573675.651395:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573675.651397:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573675.651398:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573675.651400:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573675.651402:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573675.651404:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788573675.651406:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573675.651407:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573675.651409:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788573675.651411:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573675.651413:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.651415:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573675.651418:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573675.651420:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573675.651422:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573675.651423:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573676.160942:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573676.160947:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573676.160952:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573676.160958:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573676.160963:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573676.160967:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573676.160973:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573676.160976:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573676.160979:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573677.184914:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573677.184921:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573677.184925:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573677.184933:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573677.184939:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573677.184942:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573677.184958:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573677.184961:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573677.184980:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573678.208917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573678.208923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573678.208926:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573678.208932:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573678.208938:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573678.208942:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573678.208957:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573678.208960:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573678.208962:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573679.232952:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573679.232979:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573679.232981:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573679.232985:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573679.232992:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573679.232999:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573679.233004:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573679.233008:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573679.233013:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573679.488879:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309039341, at 4309039360 00000100:00000010:0.0:1788573679.488899:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573679.936946:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573679.936951:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573679.936955:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573680.769175:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573680.769181:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573680.769186:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573680.769199:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516602752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.769220:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573680.769223:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.769230:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573680.769235:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573680.769239:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573680.769244:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516602752:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573680.769263:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573680.769268:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573680.769274:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.769279:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573680.769285:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573680.769289:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573680.769296:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573680.769302:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573680.769310:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573680.769332:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516602752, portal 25 00000100:00000001:0.0:1788573680.769337:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573680.769340:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788573680.769350:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516602752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.769364:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573680.769367:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573680.769372:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573680.769378:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516602752, offset 0 00000400:00000200:0.0:1788573680.769390:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573680.769401:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573680.769415:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573680.769422:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573680.769429:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573680.769435:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573680.769441:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573680.769450:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546361:546361:256:4294967295] 192.168.202.128@tcp LPNI seq info [546284:546284:8:4294967295] 00000400:00000200:0.0:1788573680.769459:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573680.769475:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573680.769483:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573680.769492:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573680.769500:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573680.769511:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788573680.769531:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573680.769534:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573680.769542:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573680.769547:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573680.769551:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573680.769555:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573680.769560:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573680.769566:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516602752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.769578:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.769584:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516602752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.769595:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.769598:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573680.769602:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573680.769610:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573680.769614:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309044461 00000100:00000010:0.0:1788573680.769623:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573680.769632:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573680.769642:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309044461 00000100:00000010:0.0:1788573680.769650:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50ba0. 00000100:00000001:0.0:1788573680.769656:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573680.769663:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573680.769668:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.769672:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573680.769680:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573680/real 1788573680] req@ffff8fc4ac820000 x1875440516602752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573680.769699:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573680.769703:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573680.769712:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573680.769717:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573680.769727:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573680.769734:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573680.769741:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516602752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573680.769764:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516602752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.769775:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573680.769779:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.769785:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.769790:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573680.769798:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516602752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.769853:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573680.769862:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516602752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.769887:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573680.769893:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.769894:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.769895:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.769898:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573680.769900:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573680.769905:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573680.769913:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573680.769917:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573680.769933:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516602752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573680.769955:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516602752:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573680.769978:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.769984:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.769993:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516602752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770012:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573680.770016:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573680.770019:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573680.770024:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573680.770028:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573680.770032:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573680.770038:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573680.770044:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788573680.770050:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573680.770054:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573680.770059:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788573680.770067:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573680.770070:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.770073:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.770084:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573680.770088:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.770093:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573680.770097:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573680.770100:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573680.770107:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516602880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770119:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573680.770125:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.770132:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573680.770137:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573680.770146:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573680.770155:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516602880:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573680.770165:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573680.770170:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573680.770174:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.770185:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573680.770191:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573680.770194:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573680.770199:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573680.770203:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573680.770206:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573680.770209:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516602880, portal 10 00000100:00000001:0.0:1788573680.770212:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573680.770213:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788573680.770222:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516602880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770230:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573680.770233:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573680.770236:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573680.770238:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516602880, offset 0 00000400:00000200:0.0:1788573680.770243:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573680.770247:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573680.770251:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573680.770255:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573680.770258:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573680.770260:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573680.770263:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573680.770266:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546362:546362:256:4294967295] 192.168.202.128@tcp LPNI seq info [546285:546285:8:4294967295] 00000400:00000200:0.0:1788573680.770271:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573680.770278:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573680.770283:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573680.770287:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573680.770291:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573680.770294:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788573680.770307:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573680.770309:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573680.770335:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573680.770340:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573680.770344:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573680.770347:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573680.770350:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573680.770354:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516602880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770363:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.770367:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516602880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770374:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.770375:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573680.770380:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573680.770384:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573680.770388:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573680.770390:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309040659 00000100:00000001:0.0:1788573680.770393:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309044461 00000100:00000001:0.0:1788573680.770396:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309040659 00000100:00000001:0.0:1788573680.770398:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309044461 00000100:00000001:0.0:1788573680.770401:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573680.770403:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.770405:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573680.770410:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573680/real 1788573680] req@ffff8fc4ac823b80 x1875440516602880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573680.770419:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573680.770421:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573680.770423:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573680.770426:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573680.770427:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573680.770430:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573680.770444:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516602880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573680.770454:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516602880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770460:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573680.770462:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.770464:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.770466:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573680.770470:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516602880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770478:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573680.770482:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516602880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770490:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573680.770493:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.770495:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.770496:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.770498:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573680.770500:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573680.770503:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573680.770507:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573680.770509:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573680.770514:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516602880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573680.770523:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516602880:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573680.770527:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573680.770531:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516603008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770557:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573680.770558:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.770560:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573680.770561:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573680.770563:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573680.770565:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603008:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573680.770568:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573680.770569:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573680.770570:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.770572:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573680.770574:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573680.770575:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573680.770578:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573680.770580:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573680.770583:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573680.770585:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516603008, portal 10 00000100:00000001:0.0:1788573680.770586:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573680.770587:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788573680.770592:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516603008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770596:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573680.770598:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573680.770599:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573680.770601:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516603008, offset 0 00000400:00000200:0.0:1788573680.770603:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573680.770606:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573680.770609:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573680.770611:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573680.770613:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573680.770615:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573680.770616:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573680.770622:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546363:546363:256:4294967295] 192.168.202.128@tcp LPNI seq info [546286:546286:8:4294967295] 00000400:00000200:0.0:1788573680.770625:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573680.770630:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573680.770633:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573680.770636:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573680.770639:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573680.770641:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788573680.770648:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573680.770650:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573680.770652:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573680.770654:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573680.770657:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573680.770658:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573680.770660:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573680.770662:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516603008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770668:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.770670:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516603008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770674:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.770675:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573680.770677:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573680.770679:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573680.770681:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573680.770682:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309040666 00000100:00000001:0.0:1788573680.770684:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309044461 00000100:00000001:0.0:1788573680.770685:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309040666 00000100:00000001:0.0:1788573680.770687:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309044461 00000100:00000001:0.0:1788573680.770689:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573680.770693:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.770694:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573680.770697:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573680/real 1788573680] req@ffff8fc4ac821500 x1875440516603008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573680.770703:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573680.770704:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573680.770705:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573680.770707:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573680.770708:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573680.770709:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573680.770711:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516603008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573680.770717:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516603008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770721:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573680.770722:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.770723:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.770724:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573680.770726:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516603008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770730:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573680.770733:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516603008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770737:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573680.770738:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.770739:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.770740:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.770741:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573680.770743:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573680.770744:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573680.770749:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573680.770751:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573680.770754:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516603008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573680.770759:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603008:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573680.770761:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573680.770763:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516603136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770767:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573680.770768:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.770770:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573680.770771:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573680.770772:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573680.770775:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603136:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573680.770778:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573680.770780:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573680.770781:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.770783:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573680.770786:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573680.770788:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573680.770792:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573680.770795:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573680.770799:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573680.770802:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516603136, portal 4 00000100:00000001:0.0:1788573680.770804:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573680.770806:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788573680.770831:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516603136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770836:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573680.770842:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573680.770844:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573680.770847:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516603136, offset 0 00000400:00000200:0.0:1788573680.770849:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573680.770852:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573680.770855:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573680.770857:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573680.770859:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573680.770861:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573680.770862:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573680.770864:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546364:546364:256:4294967295] 192.168.202.128@tcp LPNI seq info [546287:546287:8:4294967295] 00000400:00000200:0.0:1788573680.770867:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573680.770872:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573680.770874:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573680.770877:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573680.770879:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573680.770881:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788573680.770889:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573680.770890:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573680.770892:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573680.770895:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573680.770898:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573680.770900:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573680.770903:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573680.770907:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516603136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770915:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.770918:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516603136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.770925:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.770927:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573680.770939:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573680.770942:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573680.770945:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573680.770947:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309040807 00000100:00000001:0.0:1788573680.770950:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309044461 00000100:00000001:0.0:1788573680.770952:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309040807 00000100:00000001:0.0:1788573680.770954:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309044461 00000100:00000001:0.0:1788573680.770956:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573680.770958:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.770960:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573680.770964:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573680/real 1788573680] req@ffff8fc4ac823100 x1875440516603136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573680.770973:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573680.770974:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573680.770976:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573680.770979:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573680.770981:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573680.770983:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573680.770986:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516603136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573680.770996:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516603136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771003:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573680.771004:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771005:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.771006:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573680.771009:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516603136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771013:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573680.771016:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516603136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771026:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573680.771027:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771029:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.771030:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771031:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573680.771032:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573680.771034:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573680.771036:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573680.771038:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573680.771040:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516603136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573680.771045:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603136:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573680.771048:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771049:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.771051:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516602880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771055:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573680.771056:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573680.771058:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573680.771060:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573680.771061:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573680.771062:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573680.771063:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573680.771065:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788573680.771066:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573680.771068:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573680.771069:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788573680.771071:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573680.771072:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771075:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.771078:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516603008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771082:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573680.771083:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573680.771084:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573680.771086:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573680.771087:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573680.771088:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573680.771089:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573680.771090:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788573680.771091:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573680.771092:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573680.771093:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788573680.771094:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573680.771095:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771096:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.771098:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516603136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771102:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573680.771102:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573680.771103:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573680.771105:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573680.771105:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573680.771106:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573680.771107:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573680.771108:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788573680.771109:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573680.771110:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573680.771110:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788573680.771111:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573680.771112:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771113:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771116:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573680.771117:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.771118:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573680.771122:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573680.771123:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573680.771126:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516603264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771130:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573680.771131:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.771133:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573680.771134:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573680.771135:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573680.771137:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603264:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573680.771140:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573680.771142:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573680.771144:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573680.771145:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573680.771148:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573680.771150:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573680.771154:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573680.771157:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573680.771160:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573680.771164:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516603264, portal 4 00000100:00000001:0.0:1788573680.771166:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573680.771167:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788573680.771173:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516603264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771180:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573680.771183:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573680.771186:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573680.771189:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516603264, offset 0 00000400:00000200:0.0:1788573680.771192:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573680.771196:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573680.771200:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573680.771203:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573680.771206:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573680.771214:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573680.771216:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573680.771219:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546365:546365:256:4294967295] 192.168.202.128@tcp LPNI seq info [546288:546288:8:4294967295] 00000400:00000200:0.0:1788573680.771224:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573680.771231:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573680.771236:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573680.771240:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573680.771244:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573680.771247:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788573680.771256:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573680.771257:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573680.771259:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573680.771262:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573680.771265:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573680.771267:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573680.771270:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573680.771274:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516603264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771282:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.771285:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516603264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573705 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771292:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.771294:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573680.771298:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573680.771301:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573680.771305:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573680.771307:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309040809 00000100:00000001:0.0:1788573680.771309:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309044461 00000100:00000001:0.0:1788573680.771312:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309040809 00000100:00000001:0.0:1788573680.771353:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309044461 00000100:00000001:0.0:1788573680.771356:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573680.771358:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.771360:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573680.771366:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573680/real 1788573680] req@ffff8fc4ac822a00 x1875440516603264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573680.771375:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573680.771376:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573680.771378:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573680.771381:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573680.771383:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573680.771385:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573680.771388:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516603264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573680.771395:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516603264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771402:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573680.771403:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771405:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.771406:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573680.771410:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516603264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771417:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573680.771421:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516603264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771428:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573680.771431:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771433:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573680.771434:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771441:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573680.771443:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573680.771446:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573680.771450:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573680.771452:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573680.771457:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516603264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573680.771466:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603264:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573680.771470:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771472:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573680.771475:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516603264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573705 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573680.771482:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573680.771484:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573680.771486:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573680.771489:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573680.771491:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573680.771492:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573680.771494:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573680.771496:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788573680.771499:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573680.771500:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573680.771501:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788573680.771504:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573680.771505:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771507:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573680.771511:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573680.771513:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573680.771515:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573680.771517:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573681.280973:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573681.280978:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573681.280998:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573681.281002:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573681.281006:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573681.281009:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.281013:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573681.281014:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573681.281019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573682.304935:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573682.304941:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573682.304944:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573682.304949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573682.304958:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573682.304964:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573682.304978:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573682.304981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573682.304983:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573683.328939:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573683.328944:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573683.328947:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573683.328954:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573683.328959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573683.328965:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573683.328979:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573683.328983:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573683.328986:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573684.352909:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573684.352914:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573684.352919:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573684.352925:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573684.352933:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573684.352937:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573684.352959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573684.352961:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573684.352964:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573684.608923:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309044461, at 4309044480 00000100:00000010:0.0:1788573684.608945:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573685.056984:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573685.057003:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573685.057006:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573685.376977:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573685.376984:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573685.376988:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573685.376992:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573685.376996:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573685.377001:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.377014:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573685.377018:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.377021:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.889103:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573685.889108:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573685.889113:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573685.889123:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516603392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.889149:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573685.889152:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573685.889158:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573685.889163:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573685.889166:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573685.889173:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516603392:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573685.889178:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573685.889183:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573685.889187:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573685.889200:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573685.889207:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573685.889210:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573685.889217:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573685.889223:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573685.889230:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573685.889239:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516603392, portal 25 00000100:00000001:0.0:1788573685.889242:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573685.889245:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788573685.889252:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516603392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.889271:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573685.889275:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573685.889280:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573685.889282:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516603392, offset 0 00000400:00000200:0.0:1788573685.889291:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573685.889301:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573685.889309:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573685.889317:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573685.889322:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573685.889329:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573685.889334:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573685.889340:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546366:546366:256:4294967295] 192.168.202.128@tcp LPNI seq info [546289:546289:8:4294967295] 00000400:00000200:0.0:1788573685.889347:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573685.889363:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573685.889370:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573685.889377:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573685.889391:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573685.889397:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788573685.889418:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573685.889421:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573685.889429:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573685.889437:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573685.889443:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573685.889447:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573685.889455:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573685.889461:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516603392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.889475:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573685.889480:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516603392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.889517:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.889522:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573685.889531:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573685.889540:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573685.889544:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309049581 00000100:00000010:0.0:1788573685.889550:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573685.889560:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573685.889567:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309049581 00000100:00000010:0.0:1788573685.889571:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50480. 00000100:00000001:0.0:1788573685.889574:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573685.889578:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573685.889583:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.889587:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573685.889594:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573685/real 1788573685] req@ffff8fc4901a7480 x1875440516603392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573685.889608:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573685.889611:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573685.889614:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573685.889618:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573685.889621:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573685.889625:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573685.889631:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516603392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573685.889650:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516603392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.889667:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573685.889669:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.889672:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573685.889676:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573685.889683:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516603392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.889709:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573685.889719:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516603392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.889728:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573685.889734:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.889737:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573685.889738:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.889739:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573685.889742:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573685.889745:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573685.889754:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573685.889761:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573685.889771:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516603392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573685.889791:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516603392:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573685.889801:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.889833:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573685.889843:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516603392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.889856:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573685.889860:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573685.889864:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573685.889869:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573685.889872:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573685.889875:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573685.889880:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573685.889883:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7000. 02000000:00000001:0.0:1788573685.889889:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573685.889898:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573685.889901:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788573685.889907:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573685.889911:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.889915:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.889920:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573685.889923:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.889927:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573685.889931:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573685.889933:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573685.889941:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516603520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.889958:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573685.889961:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573685.889965:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573685.889967:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573685.889970:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573685.889976:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603520:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573685.889982:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573685.889985:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573685.889986:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573685.889988:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573685.889991:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573685.889994:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573685.890001:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573685.890006:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573685.890011:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573685.890014:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516603520, portal 10 00000100:00000001:0.0:1788573685.890018:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573685.890020:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788573685.890027:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516603520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890038:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573685.890043:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573685.890054:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573685.890057:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516603520, offset 0 00000400:00000200:0.0:1788573685.890063:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573685.890069:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573685.890074:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573685.890080:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573685.890084:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573685.890087:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573685.890090:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573685.890092:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546367:546367:256:4294967295] 192.168.202.128@tcp LPNI seq info [546290:546290:8:4294967295] 00000400:00000200:0.0:1788573685.890097:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573685.890110:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573685.890118:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573685.890123:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573685.890134:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573685.890137:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788573685.890157:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573685.890159:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573685.890162:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573685.890167:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573685.890169:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573685.890172:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573685.890175:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573685.890178:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516603520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890184:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573685.890201:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516603520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890209:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.890210:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573685.890221:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573685.890224:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573685.890228:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573685.890230:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309045779 00000100:00000001:0.0:1788573685.890233:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309049581 00000100:00000001:0.0:1788573685.890235:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309045779 00000100:00000001:0.0:1788573685.890237:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309049581 00000100:00000001:0.0:1788573685.890238:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573685.890240:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.890242:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573685.890245:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573685/real 1788573685] req@ffff8fc4901a5880 x1875440516603520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573685.890251:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573685.890252:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573685.890254:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573685.890255:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573685.890257:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573685.890258:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573685.890260:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516603520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573685.890266:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516603520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890271:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573685.890272:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890273:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573685.890274:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573685.890276:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516603520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890280:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573685.890282:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516603520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890290:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573685.890292:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890293:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573685.890294:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890295:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573685.890296:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573685.890298:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573685.890300:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573685.890302:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573685.890304:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516603520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573685.890310:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603520:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573685.890312:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573685.890314:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516603648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890318:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573685.890319:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573685.890321:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573685.890322:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573685.890323:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573685.890325:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603648:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573685.890327:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573685.890328:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573685.890329:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573685.890330:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573685.890333:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573685.890334:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573685.890339:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573685.890341:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573685.890343:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573685.890345:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516603648, portal 10 00000100:00000001:0.0:1788573685.890347:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573685.890348:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788573685.890352:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516603648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890357:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573685.890358:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573685.890360:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573685.890361:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516603648, offset 0 00000400:00000200:0.0:1788573685.890364:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573685.890366:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573685.890369:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573685.890371:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573685.890373:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573685.890374:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573685.890376:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573685.890378:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546368:546368:256:4294967295] 192.168.202.128@tcp LPNI seq info [546291:546291:8:4294967295] 00000400:00000200:0.0:1788573685.890381:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573685.890385:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573685.890388:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573685.890390:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573685.890393:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573685.890395:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788573685.890402:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573685.890403:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573685.890405:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573685.890407:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573685.890409:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573685.890414:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573685.890416:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573685.890418:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516603648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890424:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573685.890426:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516603648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890430:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.890431:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573685.890433:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573685.890435:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573685.890437:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573685.890439:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309045786 00000100:00000001:0.0:1788573685.890440:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309049581 00000100:00000001:0.0:1788573685.890442:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309045786 00000100:00000001:0.0:1788573685.890443:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309049581 00000100:00000001:0.0:1788573685.890445:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573685.890446:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.890447:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573685.890449:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573685/real 1788573685] req@ffff8fc4901a4700 x1875440516603648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573685.890454:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573685.890455:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573685.890456:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573685.890458:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573685.890459:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573685.890460:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573685.890462:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516603648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573685.890467:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516603648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890474:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573685.890475:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890476:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573685.890477:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573685.890480:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516603648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890484:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573685.890487:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516603648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890491:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573685.890493:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890494:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573685.890494:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890496:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573685.890497:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573685.890499:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573685.890501:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573685.890503:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573685.890506:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516603648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573685.890514:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603648:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573685.890518:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573685.890521:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516603776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890528:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573685.890530:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573685.890537:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573685.890539:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573685.890542:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573685.890545:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603776:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573685.890549:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573685.890551:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573685.890552:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573685.890554:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573685.890557:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573685.890558:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573685.890561:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573685.890563:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573685.890566:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573685.890568:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516603776, portal 4 00000100:00000001:0.0:1788573685.890569:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573685.890570:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788573685.890574:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516603776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890579:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573685.890581:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573685.890582:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573685.890583:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516603776, offset 0 00000400:00000200:0.0:1788573685.890586:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573685.890588:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573685.890591:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573685.890593:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573685.890595:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573685.890596:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573685.890598:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573685.890600:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546369:546369:256:4294967295] 192.168.202.128@tcp LPNI seq info [546292:546292:8:4294967295] 00000400:00000200:0.0:1788573685.890603:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573685.890607:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573685.890616:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573685.890618:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573685.890620:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573685.890622:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788573685.890629:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573685.890631:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573685.890633:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573685.890635:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573685.890637:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573685.890639:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573685.890641:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573685.890643:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516603776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890648:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573685.890650:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516603776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890655:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.890656:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573685.890658:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573685.890660:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573685.890662:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573685.890663:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309045926 00000100:00000001:0.0:1788573685.890665:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309049581 00000100:00000001:0.0:1788573685.890667:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309045926 00000100:00000001:0.0:1788573685.890668:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309049581 00000100:00000001:0.0:1788573685.890669:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573685.890670:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.890672:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573685.890674:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573685/real 1788573685] req@ffff8fc4901a4e00 x1875440516603776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573685.890683:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573685.890684:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573685.890686:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573685.890687:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573685.890688:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573685.890690:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573685.890692:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516603776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573685.890697:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516603776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890701:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573685.890702:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890703:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573685.890704:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573685.890706:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516603776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890711:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573685.890713:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516603776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890717:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573685.890718:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890719:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573685.890720:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890721:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573685.890723:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573685.890724:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573685.890726:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573685.890727:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573685.890733:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516603776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573685.890739:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603776:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573685.890741:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890743:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573685.890744:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516603520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890751:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573685.890752:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573685.890754:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573685.890757:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573685.890759:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573685.890760:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573685.890763:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573685.890764:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6400. 02000000:00000001:0.0:1788573685.890768:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573685.890769:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573685.890771:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788573685.890776:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573685.890778:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890779:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573685.890784:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516603648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890791:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573685.890793:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573685.890795:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573685.890797:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573685.890799:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573685.890800:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573685.890802:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573685.890804:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788573685.890806:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573685.890832:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573685.890834:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788573685.890839:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573685.890840:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890843:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573685.890847:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516603776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890852:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573685.890854:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573685.890855:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573685.890857:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573685.890858:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573685.890859:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573685.890860:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573685.890862:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788573685.890863:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573685.890864:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573685.890865:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788573685.890867:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573685.890868:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890869:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.890872:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573685.890873:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.890875:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573685.890876:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573685.890877:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573685.890880:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516603904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890885:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573685.890886:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573685.890888:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573685.890889:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573685.890890:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573685.890892:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603904:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573685.890894:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573685.890899:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573685.890900:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573685.890902:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573685.890903:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573685.890905:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573685.890908:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573685.890909:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573685.890912:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573685.890914:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516603904, portal 4 00000100:00000001:0.0:1788573685.890915:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573685.890916:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788573685.890920:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516603904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890925:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573685.890926:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573685.890927:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573685.890929:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516603904, offset 0 00000400:00000200:0.0:1788573685.890931:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573685.890934:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573685.890936:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573685.890938:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573685.890940:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573685.890942:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573685.890943:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573685.890945:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546370:546370:256:4294967295] 192.168.202.128@tcp LPNI seq info [546293:546293:8:4294967295] 00000400:00000200:0.0:1788573685.890948:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573685.890952:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573685.890955:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573685.890958:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573685.890960:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573685.890962:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788573685.890969:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573685.890974:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573685.890976:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573685.890978:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573685.890981:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573685.890982:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573685.890984:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573685.890986:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516603904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890991:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573685.890993:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516603904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573710 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.890998:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.890999:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573685.891001:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573685.891003:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573685.891005:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573685.891006:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309045929 00000100:00000001:0.0:1788573685.891008:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309049581 00000100:00000001:0.0:1788573685.891009:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309045929 00000100:00000001:0.0:1788573685.891011:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309049581 00000100:00000001:0.0:1788573685.891013:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573685.891014:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.891016:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573685.891020:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573685/real 1788573685] req@ffff8fc4901a6300 x1875440516603904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573685.891028:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573685.891029:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573685.891031:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573685.891035:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573685.891036:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573685.891044:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573685.891048:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516603904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573685.891057:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516603904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.891065:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573685.891066:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.891068:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573685.891070:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573685.891073:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516603904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.891079:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573685.891083:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516603904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.891089:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573685.891091:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.891092:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573685.891093:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.891095:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573685.891097:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573685.891099:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573685.891103:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573685.891105:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573685.891108:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516603904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573685.891115:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516603904:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573685.891118:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.891120:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573685.891130:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516603904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573710 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573685.891137:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573685.891138:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573685.891141:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573685.891143:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573685.891144:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573685.891146:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573685.891148:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573685.891149:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788573685.891152:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573685.891153:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573685.891154:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788573685.891159:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573685.891160:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.891163:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573685.891166:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573685.891167:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573685.891168:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573685.891170:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573686.400919:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573686.400928:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573686.400933:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573686.400940:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573686.400946:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573686.400953:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573686.400965:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573686.400969:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573686.400973:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573687.424894:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573687.424904:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573687.424909:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573687.424914:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573687.424920:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573687.424925:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573687.424937:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573687.424942:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573687.424951:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573687.425012:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573687.425016:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573687.425018:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573687.425021:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573687.425022:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573687.425024:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573687.425025:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573687.425026:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573687.425028:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573688.448931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573688.448935:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573688.448939:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573688.448944:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573688.448951:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573688.448956:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573688.448974:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573688.448976:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573688.448979:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573688.961108:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788573689.472936:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573689.472942:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573689.472947:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573689.472952:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573689.472959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573689.472964:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573689.472980:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573689.472985:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573689.472999:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573689.728877:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309049581, at 4309049600 00000100:00000010:0.0:1788573689.728895:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573690.176976:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573690.176981:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573690.176985:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573690.496925:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573690.496933:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573690.496938:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573690.496961:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573690.496969:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573690.496974:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573690.496985:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573690.496989:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573690.496993:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.009163:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573691.009171:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573691.009176:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573691.009189:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516604032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.009204:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573691.009207:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573691.009212:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573691.009216:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573691.009221:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573691.009228:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516604032:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573691.009236:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573691.009240:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573691.009245:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573691.009250:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573691.009255:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573691.009260:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573691.009271:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573691.009277:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573691.009286:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573691.009294:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516604032, portal 25 00000100:00000001:0.0:1788573691.009297:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573691.009301:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788573691.009312:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516604032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.009329:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573691.009336:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573691.009340:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573691.009352:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516604032, offset 0 00000400:00000200:0.0:1788573691.009365:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573691.009376:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573691.009383:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573691.009391:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573691.009399:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573691.009403:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573691.009406:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573691.009411:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546371:546371:256:4294967295] 192.168.202.128@tcp LPNI seq info [546294:546294:8:4294967295] 00000400:00000200:0.0:1788573691.009420:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573691.009434:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573691.009441:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573691.009447:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573691.009458:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573691.009465:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573691.009488:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573691.009491:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573691.009500:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573691.009511:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573691.009516:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573691.009520:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573691.009527:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573691.009534:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516604032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.009544:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573691.009550:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516604032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.009562:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.009566:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573691.009571:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573691.009580:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573691.009591:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309054701 00000100:00000010:0.0:1788573691.009599:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573691.009606:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573691.009614:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309054701 00000100:00000010:0.0:1788573691.009620:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50420. 00000100:00000001:0.0:1788573691.009625:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573691.009629:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573691.009634:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.009639:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573691.009646:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573691/real 1788573691] req@ffff8fc4901a6300 x1875440516604032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573691.009658:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573691.009661:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573691.009664:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573691.009669:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573691.009672:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573691.009675:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573691.009682:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516604032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573691.009700:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516604032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.009714:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573691.009718:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.009722:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573691.009725:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573691.009735:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516604032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.009757:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573691.009769:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516604032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.009826:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573691.009829:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.009831:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573691.009833:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.009838:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573691.009841:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573691.009847:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573691.009855:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573691.009859:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573691.009871:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516604032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573691.009891:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516604032:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573691.009896:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.009900:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573691.009905:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516604032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.009913:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573691.009916:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573691.009920:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573691.009926:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573691.009929:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573691.009931:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573691.009933:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573691.009936:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788573691.009940:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573691.009942:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573691.009943:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788573691.009947:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573691.009948:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.009950:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.009960:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573691.009962:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.009963:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573691.009964:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573691.009965:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573691.009969:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516604160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.009975:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573691.009976:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573691.009977:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573691.009979:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573691.009981:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573691.009984:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516604160:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573691.009987:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573691.009992:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573691.009994:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573691.009995:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573691.009998:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573691.010000:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573691.010004:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573691.010006:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573691.010009:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573691.010011:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516604160, portal 10 00000100:00000001:0.0:1788573691.010013:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573691.010014:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788573691.010020:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516604160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010025:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573691.010026:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573691.010028:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573691.010029:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516604160, offset 0 00000400:00000200:0.0:1788573691.010032:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573691.010035:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573691.010042:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573691.010044:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573691.010046:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573691.010048:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573691.010050:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573691.010052:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546372:546372:256:4294967295] 192.168.202.128@tcp LPNI seq info [546295:546295:8:4294967295] 00000400:00000200:0.0:1788573691.010055:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573691.010073:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573691.010079:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573691.010082:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573691.010086:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573691.010089:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573691.010101:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573691.010102:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573691.010104:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573691.010107:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573691.010109:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573691.010111:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573691.010113:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573691.010116:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516604160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010121:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573691.010123:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516604160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010128:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.010129:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573691.010131:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573691.010134:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573691.010137:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573691.010139:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309050899 00000100:00000001:0.0:1788573691.010147:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309054701 00000100:00000001:0.0:1788573691.010149:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309050899 00000100:00000001:0.0:1788573691.010151:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309054701 00000100:00000001:0.0:1788573691.010153:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573691.010155:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.010157:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573691.010162:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573691/real 1788573691] req@ffff8fc4901a4e00 x1875440516604160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573691.010171:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573691.010173:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573691.010175:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573691.010178:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573691.010180:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573691.010183:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573691.010186:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516604160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573691.010194:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516604160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010198:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573691.010199:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010201:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573691.010202:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573691.010204:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516604160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010209:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573691.010211:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516604160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010215:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573691.010217:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010222:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573691.010223:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010225:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573691.010226:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573691.010228:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573691.010231:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573691.010232:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573691.010235:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516604160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573691.010240:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516604160:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573691.010243:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573691.010245:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516604288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010249:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573691.010250:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573691.010251:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573691.010252:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573691.010254:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573691.010255:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516604288:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573691.010257:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573691.010259:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573691.010260:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573691.010261:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573691.010263:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573691.010265:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573691.010267:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573691.010269:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573691.010271:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573691.010273:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516604288, portal 10 00000100:00000001:0.0:1788573691.010275:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573691.010278:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788573691.010283:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516604288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010287:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573691.010289:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573691.010290:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573691.010292:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516604288, offset 0 00000400:00000200:0.0:1788573691.010295:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573691.010297:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573691.010300:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573691.010302:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573691.010303:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573691.010305:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573691.010307:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573691.010309:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546373:546373:256:4294967295] 192.168.202.128@tcp LPNI seq info [546296:546296:8:4294967295] 00000400:00000200:0.0:1788573691.010312:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573691.010316:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573691.010320:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573691.010322:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573691.010325:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573691.010326:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573691.010333:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573691.010335:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573691.010337:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573691.010339:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573691.010341:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573691.010342:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573691.010344:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573691.010346:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516604288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010358:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573691.010361:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516604288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010365:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.010366:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573691.010369:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573691.010371:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573691.010373:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573691.010374:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309050906 00000100:00000001:0.0:1788573691.010376:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309054701 00000100:00000001:0.0:1788573691.010378:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309050906 00000100:00000001:0.0:1788573691.010379:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309054701 00000100:00000001:0.0:1788573691.010380:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573691.010381:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.010383:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573691.010385:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573691/real 1788573691] req@ffff8fc4901a4700 x1875440516604288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573691.010390:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573691.010391:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573691.010392:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573691.010393:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573691.010394:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573691.010396:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573691.010398:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516604288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573691.010403:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516604288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010406:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573691.010410:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010411:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573691.010412:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573691.010415:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516604288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010419:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573691.010422:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516604288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010426:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573691.010427:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010428:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573691.010429:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010431:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573691.010432:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573691.010434:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573691.010436:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573691.010437:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573691.010439:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516604288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573691.010444:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516604288:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573691.010446:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573691.010448:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516604416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010452:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573691.010453:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573691.010454:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573691.010455:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573691.010456:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573691.010461:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516604416:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573691.010463:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573691.010465:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573691.010466:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573691.010467:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573691.010469:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573691.010470:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573691.010472:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573691.010474:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573691.010476:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573691.010478:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516604416, portal 4 00000100:00000001:0.0:1788573691.010479:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573691.010480:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788573691.010484:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516604416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010488:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573691.010490:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573691.010491:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573691.010492:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516604416, offset 0 00000400:00000200:0.0:1788573691.010495:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573691.010497:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573691.010499:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573691.010501:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573691.010502:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573691.010504:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573691.010505:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573691.010507:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546374:546374:256:4294967295] 192.168.202.128@tcp LPNI seq info [546297:546297:8:4294967295] 00000400:00000200:0.0:1788573691.010510:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573691.010514:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573691.010517:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573691.010519:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573691.010525:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573691.010527:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573691.010534:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573691.010535:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573691.010537:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573691.010539:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573691.010542:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573691.010543:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573691.010544:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573691.010547:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516604416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010552:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573691.010554:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516604416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010558:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.010559:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573691.010561:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573691.010563:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573691.010565:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573691.010567:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309051046 00000100:00000001:0.0:1788573691.010569:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309054701 00000100:00000001:0.0:1788573691.010570:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309051046 00000100:00000001:0.0:1788573691.010571:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309054701 00000100:00000001:0.0:1788573691.010573:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573691.010574:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.010575:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573691.010577:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573691/real 1788573691] req@ffff8fc4901a5880 x1875440516604416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573691.010582:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573691.010586:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573691.010587:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573691.010588:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573691.010589:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573691.010591:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573691.010593:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516604416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573691.010599:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516604416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010603:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573691.010604:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010605:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573691.010606:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573691.010608:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516604416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010612:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573691.010614:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516604416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010618:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573691.010620:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010621:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573691.010622:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010623:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573691.010624:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573691.010625:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573691.010627:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573691.010628:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573691.010631:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516604416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573691.010642:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516604416:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573691.010644:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573691.010646:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516604544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010650:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573691.010651:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573691.010652:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573691.010653:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573691.010654:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573691.010657:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516604544:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573691.010658:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573691.010659:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573691.010660:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573691.010661:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573691.010663:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573691.010665:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573691.010667:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573691.010668:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573691.010670:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573691.010672:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516604544, portal 4 00000100:00000001:0.0:1788573691.010673:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573691.010674:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788573691.010678:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516604544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010682:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573691.010683:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573691.010685:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573691.010686:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516604544, offset 0 00000400:00000200:0.0:1788573691.010688:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573691.010694:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573691.010696:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573691.010698:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573691.010700:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573691.010701:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573691.010703:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573691.010705:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546375:546375:256:4294967295] 192.168.202.128@tcp LPNI seq info [546298:546298:8:4294967295] 00000400:00000200:0.0:1788573691.010707:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573691.010711:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573691.010714:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573691.010715:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573691.010718:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573691.010720:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573691.010726:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573691.010728:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573691.010731:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573691.010735:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573691.010738:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573691.010740:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573691.010742:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573691.010746:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516604544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010755:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573691.010758:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516604544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573716 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010766:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.010767:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573691.010770:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573691.010773:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573691.010777:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573691.010784:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309051048 00000100:00000001:0.0:1788573691.010786:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309054701 00000100:00000001:0.0:1788573691.010787:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309051048 00000100:00000001:0.0:1788573691.010789:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309054701 00000100:00000001:0.0:1788573691.010790:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573691.010792:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.010793:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573691.010796:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573691/real 1788573691] req@ffff8fc4901a7480 x1875440516604544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573691.010801:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573691.010802:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573691.010804:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573691.010805:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573691.010821:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573691.010824:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573691.010827:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516604544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573691.010832:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516604544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010836:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573691.010837:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010839:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573691.010840:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573691.010842:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516604544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010846:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573691.010849:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516604544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010853:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573691.010858:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010859:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573691.010860:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010862:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573691.010863:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573691.010865:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573691.010867:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573691.010869:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573691.010871:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516604544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573691.010876:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516604544:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573691.010878:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010879:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573691.010881:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516604160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010885:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573691.010887:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573691.010888:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573691.010890:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573691.010891:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573691.010892:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573691.010894:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573691.010895:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788573691.010897:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573691.010898:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573691.010899:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788573691.010904:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573691.010905:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010906:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573691.010908:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516604288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010919:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573691.010920:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573691.010921:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573691.010923:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573691.010923:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573691.010924:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573691.010925:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573691.010926:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788573691.010927:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573691.010928:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573691.010929:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788573691.010930:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573691.010931:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010932:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573691.010934:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516604416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010938:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573691.010938:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573691.010939:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573691.010940:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573691.010941:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573691.010942:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573691.010943:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573691.010944:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788573691.010945:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573691.010945:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573691.010946:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788573691.010948:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573691.010948:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010949:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573691.010951:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516604544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573716 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573691.010955:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573691.010955:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573691.010960:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788573691.010961:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573691.010962:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573691.010963:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573691.010964:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573691.010965:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6400. 02000000:00000001:0.0:1788573691.010966:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573691.010967:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573691.010968:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788573691.010970:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573691.010971:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010972:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.010975:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573691.010975:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.010977:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573691.010978:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573691.520912:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573691.520921:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573691.520926:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573691.520936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573691.520944:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573691.520956:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.520992:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573691.520995:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573691.520998:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573692.544968:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573692.544978:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573692.544982:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573692.544991:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573692.545002:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573692.545019:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573692.545049:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573692.545052:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573692.545059:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573692.545082:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573692.545085:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573692.545090:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573692.545096:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573692.545116:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573692.545119:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573692.545121:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573692.545122:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573692.545124:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573693.568876:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573693.568884:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573693.568887:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573693.568895:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573693.568901:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573693.568905:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573693.568909:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573693.568913:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573693.568918:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573693.568966:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573693.568970:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573693.568975:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573693.568979:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573693.568984:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573693.568986:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573693.568989:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573693.568990:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573693.568993:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573694.848920:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309054701, at 4309054720 00000100:00000010:0.0:1788573694.848935:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573695.297034:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573695.297038:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573695.297040:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573695.616978:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573695.616984:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573695.616989:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573695.616995:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573695.617003:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573695.617007:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573695.617010:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573695.617012:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573695.617015:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.129202:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573696.129208:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573696.129211:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573696.129222:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516604672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.129243:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573696.129247:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.129254:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573696.129259:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573696.129266:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573696.129284:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516604672:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573696.129294:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573696.129301:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573696.129305:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.129310:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573696.129318:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573696.129323:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573696.129333:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573696.129340:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573696.129349:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573696.129358:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516604672, portal 25 00000100:00000001:0.0:1788573696.129363:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573696.129366:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788573696.129376:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516604672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.129387:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573696.129395:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573696.129400:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573696.129406:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516604672, offset 0 00000400:00000200:0.0:1788573696.129417:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573696.129445:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573696.129458:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573696.129467:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573696.129484:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573696.129492:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573696.129496:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573696.129503:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546376:546376:256:4294967295] 192.168.202.128@tcp LPNI seq info [546299:546299:8:4294967295] 00000400:00000200:0.0:1788573696.129512:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573696.129527:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573696.129537:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573696.129552:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573696.129564:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573696.129572:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573696.129600:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573696.129604:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573696.129614:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573696.129621:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573696.129627:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573696.129630:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573696.129636:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573696.129643:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516604672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.129658:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.129669:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516604672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.129692:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.129696:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573696.129702:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573696.129709:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573696.129712:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309059821 00000100:00000010:0.0:1788573696.129717:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573696.129725:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573696.129732:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309059821 00000100:00000010:0.0:1788573696.129742:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b506c0. 00000100:00000001:0.0:1788573696.129746:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573696.129750:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573696.129752:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.129754:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573696.129760:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573696/real 1788573696] req@ffff8fc4901a7100 x1875440516604672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573696.129777:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573696.129780:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573696.129784:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573696.129788:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573696.129792:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573696.129796:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573696.129801:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516604672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573696.129840:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516604672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.129856:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573696.129860:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.129867:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.129871:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573696.129881:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516604672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.129890:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573696.129900:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516604672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.129908:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573696.129913:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.129918:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.129920:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.129929:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573696.129944:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573696.129950:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573696.129956:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573696.129961:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573696.129969:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516604672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573696.129983:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516604672:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573696.129993:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.129998:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.130004:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516604672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130029:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573696.130033:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573696.130037:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573696.130043:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573696.130048:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573696.130051:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573696.130057:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573696.130061:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7000. 02000000:00000001:0.0:1788573696.130069:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573696.130074:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573696.130078:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788573696.130095:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573696.130099:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130104:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130114:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573696.130117:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.130120:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573696.130123:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573696.130126:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573696.130132:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516604800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130154:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573696.130158:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.130163:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573696.130167:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573696.130173:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573696.130181:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516604800:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573696.130191:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573696.130197:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573696.130200:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.130205:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573696.130211:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573696.130214:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573696.130221:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573696.130226:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573696.130232:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573696.130236:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516604800, portal 10 00000100:00000001:0.0:1788573696.130239:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573696.130242:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788573696.130253:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516604800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130262:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573696.130267:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573696.130270:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573696.130273:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516604800, offset 0 00000400:00000200:0.0:1788573696.130282:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573696.130290:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573696.130295:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573696.130300:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573696.130306:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573696.130310:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573696.130315:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573696.130325:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546377:546377:256:4294967295] 192.168.202.128@tcp LPNI seq info [546300:546300:8:4294967295] 00000400:00000200:0.0:1788573696.130331:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573696.130340:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573696.130345:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573696.130347:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573696.130351:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573696.130354:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573696.130365:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573696.130367:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573696.130371:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573696.130375:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573696.130378:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573696.130380:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573696.130383:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573696.130388:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516604800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130396:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.130400:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516604800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130405:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.130406:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573696.130409:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573696.130412:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573696.130415:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573696.130417:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309056019 00000100:00000001:0.0:1788573696.130419:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309059821 00000100:00000001:0.0:1788573696.130421:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309056019 00000100:00000001:0.0:1788573696.130423:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309059821 00000100:00000001:0.0:1788573696.130425:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573696.130430:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.130431:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573696.130434:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573696/real 1788573696] req@ffff8fc4901a5c00 x1875440516604800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573696.130440:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573696.130441:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573696.130443:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573696.130444:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573696.130445:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573696.130447:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573696.130449:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516604800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573696.130454:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516604800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130458:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573696.130459:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130460:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.130461:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573696.130464:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516604800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130467:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573696.130470:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516604800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130474:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573696.130476:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130477:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.130477:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130479:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573696.130480:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573696.130482:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573696.130487:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573696.130488:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573696.130491:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516604800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573696.130496:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516604800:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573696.130498:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573696.130500:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516604928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130504:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573696.130506:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.130507:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573696.130508:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573696.130510:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573696.130512:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516604928:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573696.130513:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573696.130515:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573696.130516:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.130517:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573696.130519:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573696.130520:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573696.130522:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573696.130524:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573696.130526:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573696.130528:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516604928, portal 10 00000100:00000001:0.0:1788573696.130529:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573696.130530:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788573696.130534:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516604928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130538:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573696.130543:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573696.130544:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573696.130546:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516604928, offset 0 00000400:00000200:0.0:1788573696.130550:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573696.130554:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573696.130558:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573696.130561:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573696.130564:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573696.130566:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573696.130569:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573696.130571:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546378:546378:256:4294967295] 192.168.202.128@tcp LPNI seq info [546301:546301:8:4294967295] 00000400:00000200:0.0:1788573696.130576:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573696.130583:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573696.130587:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573696.130591:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573696.130595:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573696.130598:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573696.130615:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573696.130616:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573696.130619:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573696.130621:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573696.130624:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573696.130625:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573696.130627:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573696.130630:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516604928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130635:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.130637:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516604928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130642:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.130646:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573696.130648:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573696.130651:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573696.130653:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573696.130655:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309056026 00000100:00000001:0.0:1788573696.130657:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309059821 00000100:00000001:0.0:1788573696.130658:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309056026 00000100:00000001:0.0:1788573696.130660:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309059821 00000100:00000001:0.0:1788573696.130661:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573696.130662:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.130664:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573696.130666:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573696/real 1788573696] req@ffff8fc4901a5180 x1875440516604928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573696.130672:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573696.130673:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573696.130674:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573696.130676:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573696.130677:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573696.130678:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573696.130680:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516604928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573696.130686:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516604928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130689:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573696.130690:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130691:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.130692:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573696.130694:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516604928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130698:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573696.130704:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516604928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130709:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573696.130710:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130712:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.130712:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130713:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573696.130715:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573696.130717:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573696.130719:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573696.130720:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573696.130722:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516604928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573696.130727:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516604928:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573696.130730:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573696.130732:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516605056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130736:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573696.130737:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.130739:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573696.130740:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573696.130741:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573696.130743:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516605056:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573696.130745:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573696.130746:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573696.130747:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.130748:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573696.130751:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573696.130752:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573696.130758:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573696.130759:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573696.130762:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573696.130764:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516605056, portal 4 00000100:00000001:0.0:1788573696.130765:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573696.130766:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788573696.130770:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516605056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130774:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573696.130776:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573696.130777:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573696.130779:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516605056, offset 0 00000400:00000200:0.0:1788573696.130781:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573696.130784:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573696.130786:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573696.130788:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573696.130789:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573696.130791:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573696.130792:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573696.130794:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546379:546379:256:4294967295] 192.168.202.128@tcp LPNI seq info [546302:546302:8:4294967295] 00000400:00000200:0.0:1788573696.130797:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573696.130801:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573696.130804:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573696.130806:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573696.130825:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573696.130827:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573696.130834:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573696.130835:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573696.130838:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573696.130840:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573696.130842:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573696.130847:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573696.130849:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573696.130852:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516605056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130857:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.130859:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516605056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130864:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.130865:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573696.130868:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573696.130870:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573696.130872:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573696.130873:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309056167 00000100:00000001:0.0:1788573696.130875:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309059821 00000100:00000001:0.0:1788573696.130877:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309056167 00000100:00000001:0.0:1788573696.130878:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309059821 00000100:00000001:0.0:1788573696.130879:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573696.130880:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.130882:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573696.130884:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573696/real 1788573696] req@ffff8fc4901a7800 x1875440516605056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573696.130889:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573696.130890:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573696.130891:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573696.130893:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573696.130894:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573696.130895:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573696.130897:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516605056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573696.130903:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516605056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130910:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573696.130911:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130913:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.130914:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573696.130916:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516605056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130920:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573696.130923:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516605056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130927:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573696.130929:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130930:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.130947:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130949:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573696.130951:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573696.130952:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573696.130955:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573696.130956:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573696.130959:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516605056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573696.130965:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516605056:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573696.130967:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130968:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.130970:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516604800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.130975:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573696.130976:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573696.130981:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573696.130983:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573696.130984:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573696.130985:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573696.130987:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573696.130988:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5c00. 02000000:00000001:0.0:1788573696.130990:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573696.130991:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573696.130992:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788573696.130996:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573696.130997:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.130998:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.131000:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516604928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.131005:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573696.131006:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573696.131006:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573696.131008:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573696.131009:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573696.131010:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573696.131011:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573696.131012:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788573696.131013:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573696.131014:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573696.131015:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788573696.131016:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573696.131017:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.131018:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.131020:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516605056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.131023:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573696.131024:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573696.131025:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573696.131026:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573696.131027:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573696.131028:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573696.131033:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573696.131035:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788573696.131036:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573696.131037:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573696.131038:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788573696.131039:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573696.131040:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.131041:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.131044:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573696.131045:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.131046:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573696.131047:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573696.131048:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573696.131050:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516605184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.131054:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573696.131055:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.131057:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573696.131058:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573696.131060:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573696.131062:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516605184:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573696.131063:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573696.131065:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573696.131066:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573696.131067:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573696.131068:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573696.131069:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573696.131072:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573696.131073:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573696.131076:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573696.131078:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516605184, portal 4 00000100:00000001:0.0:1788573696.131079:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573696.131080:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788573696.131084:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516605184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.131095:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573696.131097:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573696.131098:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573696.131100:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516605184, offset 0 00000400:00000200:0.0:1788573696.131102:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573696.131105:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573696.131108:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573696.131110:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573696.131112:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573696.131113:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573696.131115:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573696.131117:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546380:546380:256:4294967295] 192.168.202.128@tcp LPNI seq info [546303:546303:8:4294967295] 00000400:00000200:0.0:1788573696.131120:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573696.131124:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573696.131127:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573696.131129:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573696.131132:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573696.131133:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573696.131141:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573696.131142:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788573696.131144:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573696.131146:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573696.131149:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9aa0 00000400:00000010:0.0:1788573696.131151:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9aa0. 00000100:00000001:0.0:1788573696.131153:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573696.131157:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516605184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.131165:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.131168:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516605184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573721 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.131186:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.131188:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573696.131193:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573696.131196:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573696.131200:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573696.131202:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309056169 00000100:00000001:0.0:1788573696.131205:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309059821 00000100:00000001:0.0:1788573696.131206:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309056169 00000100:00000001:0.0:1788573696.131208:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309059821 00000100:00000001:0.0:1788573696.131209:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573696.131211:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.131212:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573696.131215:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573696/real 1788573696] req@ffff8fc4901a4a80 x1875440516605184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573696.131220:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573696.131221:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573696.131222:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573696.131224:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573696.131225:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573696.131228:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573696.131232:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516605184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573696.131241:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516605184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.131247:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573696.131248:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.131250:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.131251:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573696.131255:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516605184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.131269:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573696.131274:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516605184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.131281:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573696.131283:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.131285:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573696.131286:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.131288:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573696.131290:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573696.131293:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573696.131297:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573696.131299:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573696.131304:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516605184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573696.131312:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516605184:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573696.131316:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.131318:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573696.131321:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516605184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573721 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573696.131328:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573696.131330:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573696.131332:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573696.131334:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573696.131336:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573696.131338:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573696.131340:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573696.131341:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788573696.131343:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573696.131348:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573696.131349:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788573696.131352:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573696.131353:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.131355:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.131357:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573696.131358:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.131360:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573696.131361:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573696.640893:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573696.640901:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573696.640905:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573696.640913:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573696.640931:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573696.640935:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.640940:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573696.640943:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573696.640951:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573697.664923:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573697.664932:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573697.664935:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573697.664943:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573697.664951:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573697.664955:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573697.664960:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573697.664964:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573697.664970:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573697.665010:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573697.665015:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573697.665017:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573697.665019:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573697.665022:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573697.665024:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573697.665026:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573697.665027:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573697.665029:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573698.688939:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573698.688951:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573698.688955:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573698.688977:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573698.688983:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573698.688990:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573698.689005:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573698.689014:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573698.689025:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573699.712936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573699.712942:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573699.712946:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573699.712952:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573699.712960:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573699.712966:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573699.712972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573699.712976:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573699.712982:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573699.713003:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573699.713008:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573699.713010:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573699.713014:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573699.713020:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573699.713023:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573699.713026:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573699.713028:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573699.713030:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573699.968940:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309059821, at 4309059840 00000100:00000010:0.0:1788573699.968958:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573700.417043:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573700.417046:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573700.417049:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573700.736918:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573700.736926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573700.736931:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573700.736938:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573700.736947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573700.736954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573700.736968:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573700.736972:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573700.736985:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.249149:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573701.249156:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573701.249162:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573701.249174:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516605312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.249186:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573701.249190:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.249195:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573701.249199:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573701.249203:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573701.249209:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516605312:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573701.249216:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573701.249220:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573701.249224:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.249229:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573701.249236:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573701.249242:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573701.249250:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573701.249254:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573701.249261:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573701.249270:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516605312, portal 25 00000100:00000001:0.0:1788573701.249275:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573701.249278:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788573701.249287:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516605312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.249304:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573701.249311:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573701.249315:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573701.249321:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516605312, offset 0 00000400:00000200:0.0:1788573701.249331:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573701.249341:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573701.249350:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573701.249368:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573701.249375:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573701.249382:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573701.249384:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573701.249390:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546381:546381:256:4294967295] 192.168.202.128@tcp LPNI seq info [546304:546304:8:4294967295] 00000400:00000200:0.0:1788573701.249401:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573701.249415:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573701.249423:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573701.249429:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573701.249440:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573701.249466:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573701.249473:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573701.249476:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309064941 00000100:00000010:0.0:1788573701.249481:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573701.249492:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573701.249500:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309064941 00000100:00000010:0.0:1788573701.249506:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50600. 00000100:00000001:0.0:1788573701.249511:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573701.249515:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573701.249521:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.249525:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.249527:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.249533:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573701.249536:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573701.249539:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573701.249542:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573701.249544:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573701.249551:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516605440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.249560:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573701.249561:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.249571:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573701.249575:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573701.249586:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573701.249592:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516605440:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573701.249598:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573701.249602:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573701.249604:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.249609:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573701.249612:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573701.249617:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573701.249623:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573701.249628:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915790. 00000400:00000010:0.0:1788573701.249635:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573701.249641:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516605440, portal 10 00000100:00000001:0.0:1788573701.249646:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573701.249648:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788573701.249657:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516605440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.249669:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573701.249674:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573701.249678:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9908. 00000100:00000200:0.0:1788573701.249681:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516605440, offset 0 00000400:00000200:0.0:1788573701.249688:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573701.249692:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573701.249695:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573701.249697:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573701.249700:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573701.249705:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573701.249708:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573701.249713:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546382:546382:255:4294967295] 192.168.202.128@tcp LPNI seq info [546305:546305:7:4294967295] 00000400:00000200:0.0:1788573701.249719:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573701.249731:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573701.249741:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573701.249748:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56000. 00000800:00000200:0.0:1788573701.249755:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573701.249764:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573701.249769:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573701.249775:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573701.249778:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309061138 00000100:00000001:0.0:1788573701.249784:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309064941 00000100:00000001:0.0:1788573701.249788:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309061138 00000100:00000001:0.0:1788573701.249794:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309064941 00000100:00000001:0.0:1788573701.249797:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573701.249849:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.249853:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573701.249858:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516605568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.249879:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573701.249888:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.249894:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573701.249899:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573701.249906:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573701.249916:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516605568:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573701.249925:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573701.249932:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573701.249936:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.249942:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573701.249948:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573701.249952:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573701.249961:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573701.249967:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573701.249973:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed97f8. 00000100:00000200:0.0:1788573701.249987:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516605568, portal 10 00000100:00000001:0.0:1788573701.249994:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573701.249999:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788573701.250011:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516605568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.250034:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573701.250044:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573701.250051:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9dd0. 00000100:00000200:0.0:1788573701.250058:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516605568, offset 0 00000400:00000200:0.0:1788573701.250071:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573701.250080:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573701.250087:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573701.250093:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573701.250100:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573701.250105:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573701.250110:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573701.250117:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546383:546383:254:4294967295] 192.168.202.128@tcp LPNI seq info [546306:546306:6:4294967295] 00000400:00000200:0.0:1788573701.250127:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573701.250141:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573701.250150:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573701.250157:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573701.250168:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573701.250183:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573701.250190:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573701.250195:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573701.250200:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309061146 00000100:00000001:0.0:1788573701.250206:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309064941 00000100:00000001:0.0:1788573701.250212:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309061146 00000100:00000001:0.0:1788573701.250217:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309064941 00000100:00000001:0.0:1788573701.250223:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573701.250238:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.250242:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.250244:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.250249:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573701.250251:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573701.250255:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573701.250256:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573701.250258:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573701.250263:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516605696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.250272:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573701.250273:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.250275:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573701.250278:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573701.250280:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573701.250283:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516605696:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573701.250287:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573701.250289:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573701.250291:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.250293:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573701.250296:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573701.250299:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573701.250303:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573701.250307:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915948. 00000400:00000010:0.0:1788573701.250312:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573701.250315:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516605696, portal 4 00000100:00000001:0.0:1788573701.250317:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573701.250319:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788573701.250326:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516605696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.250333:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573701.250335:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573701.250337:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9110. 00000100:00000200:0.0:1788573701.250340:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516605696, offset 0 00000400:00000200:0.0:1788573701.250352:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573701.250356:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573701.250360:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573701.250362:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573701.250365:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573701.250367:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573701.250370:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573701.250379:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546384:546384:253:4294967295] 192.168.202.128@tcp LPNI seq info [546307:546307:5:4294967295] 00000400:00000200:0.0:1788573701.250384:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573701.250390:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573701.250399:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573701.250403:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56c00. 00000800:00000200:0.0:1788573701.250408:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573701.250414:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573701.250418:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573701.250421:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573701.250424:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309061286 00000100:00000001:0.0:1788573701.250426:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309064941 00000100:00000001:0.0:1788573701.250428:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309061286 00000100:00000001:0.0:1788573701.250430:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309064941 00000100:00000001:0.0:1788573701.250432:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573701.250434:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.250436:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573701.250440:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516605824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.250448:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573701.250449:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.250452:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573701.250453:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573701.250455:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573701.250464:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516605824:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573701.250468:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573701.250470:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573701.250472:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573701.250474:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573701.250477:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573701.250479:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573701.250484:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573701.250487:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573701.250491:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573701.250494:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516605824, portal 4 00000100:00000001:0.0:1788573701.250496:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573701.250497:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788573701.250503:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516605824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573701.250509:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573701.250512:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573701.250514:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9a18. 00000100:00000200:0.0:1788573701.250516:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516605824, offset 0 00000400:00000200:0.0:1788573701.250521:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573701.250525:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573701.250528:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573701.250531:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573701.250533:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573701.250535:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573701.250537:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573701.250540:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546385:546385:252:4294967295] 192.168.202.128@tcp LPNI seq info [546308:546308:4:4294967295] 00000400:00000200:0.0:1788573701.250543:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573701.250549:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573701.250554:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573701.250558:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56b00. 00000800:00000200:0.0:1788573701.250568:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573701.250575:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573701.250578:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573701.250582:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573701.250584:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309061288 00000100:00000001:0.0:1788573701.250586:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309064941 00000100:00000001:0.0:1788573701.250588:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309061288 00000100:00000001:0.0:1788573701.250591:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309064941 00000100:00000001:0.0:1788573701.250594:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573701.250596:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.250599:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.250601:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.250605:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573701.250606:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573701.250609:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573701.250610:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573701.250611:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.250613:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.760930:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573701.760940:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573701.760945:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573701.760952:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573701.760959:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573701.760965:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.760972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573701.760977:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.760983:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573701.761008:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573701.761012:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573701.761014:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573701.761017:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573701.761019:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573701.761021:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.761024:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573701.761026:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573701.761037:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573702.784914:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573702.784923:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573702.784928:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573702.784938:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573702.784952:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573702.784960:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573702.784991:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573702.784994:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573702.784996:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.808945:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573703.808953:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573703.808956:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573703.808963:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573703.808970:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573703.808974:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573703.808983:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573703.808988:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573703.808998:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573703.809038:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573703.809045:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573703.809049:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573703.809058:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573703.809064:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573703.809068:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573703.809070:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573703.809071:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573703.809073:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573704.321271:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573704.321275:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573704.321279:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573704.321286:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573701/real 1788573704] req@ffff8fc4ac822d80 x1875440516605440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573704.321299:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573704.321301:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573704.321305:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915790. 00000400:00000200:0.0:1788573704.321308:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573704.321315:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573704.321318:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573704.321322:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516605440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573704.321332:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516605440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321341:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573704.321343:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321346:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573704.321348:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573704.321352:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516605440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321361:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573704.321366:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516605440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321385:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573704.321389:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321391:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573704.321393:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321396:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573704.321399:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573704.321402:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573704.321408:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573704.321411:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573704.321415:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516605440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573704.321424:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516605440:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573704.321429:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573704.321437:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573701/real 1788573704] req@ffff8fc4ac821f80 x1875440516605568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573704.321447:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573704.321449:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573704.321451:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573704.321453:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed97f8 00000400:00000010:0.0:1788573704.321455:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed97f8. 00000100:00000001:0.0:1788573704.321457:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573704.321460:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516605568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573704.321483:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516605568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321491:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573704.321493:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321495:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573704.321496:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573704.321500:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516605568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321506:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573704.321510:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516605568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321516:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573704.321519:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321521:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573704.321522:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321524:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573704.321528:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573704.321530:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573704.321534:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573704.321547:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573704.321552:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516605568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573704.321560:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516605568:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573704.321565:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573704.321568:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573701/real 1788573704] req@ffff8fc4ac823480 x1875440516605312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573704.321577:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573704.321578:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573704.321581:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573704.321583:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573704.321585:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573704.321588:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573704.321591:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516605312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573704.321598:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516605312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321604:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573704.321606:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321607:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573704.321609:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573704.321612:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516605312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321619:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573704.321622:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516605312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321628:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573704.321631:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321638:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573704.321640:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321642:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573704.321644:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573704.321647:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573704.321650:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573704.321652:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573704.321656:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516605312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573704.321664:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516605312:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573704.321668:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573704.321671:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573704.321675:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516605440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321682:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573704.321686:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573704.321694:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573704.321699:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573704.321702:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573704.321705:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573704.321708:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573704.321711:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788573704.321715:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573704.321717:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573704.321719:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788573704.321723:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573704.321725:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321764:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573704.321768:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516605568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321775:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573704.321782:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573704.321784:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573704.321786:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573704.321788:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573704.321789:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573704.321791:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573704.321793:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788573704.321796:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573704.321797:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573704.321798:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788573704.321802:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573704.321803:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321805:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573704.321877:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516605312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321884:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573704.321886:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573704.321888:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573704.321891:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573704.321893:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573704.321895:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573704.321898:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573704.321900:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862ac00. 02000000:00000001:0.0:1788573704.321903:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573704.321905:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573704.321907:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788573704.321911:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573704.321913:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321916:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573704.321918:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573704.321920:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573704.321923:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573704.321928:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573701/real 1788573704] req@ffff8fc4ac822300 x1875440516605696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573704.321938:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573704.321940:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573704.321949:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915948. 00000400:00000200:0.0:1788573704.321952:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573704.321954:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573704.321957:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573704.321960:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516605696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573704.321970:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516605696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321977:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573704.321979:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.321981:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573704.321983:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573704.321987:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516605696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.321994:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573704.321999:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516605696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.322006:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573704.322009:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.322011:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573704.322012:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.322014:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573704.322016:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573704.322019:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573704.322023:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573704.322025:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573704.322028:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516605696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573704.322033:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516605696:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573704.322041:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573704.322046:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573701/real 1788573704] req@ffff8fc4ac820700 x1875440516605824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573704.322055:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573704.322056:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573704.322058:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573704.322061:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573704.322063:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573704.322065:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573704.322068:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516605824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573704.322076:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516605824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.322082:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573704.322083:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.322086:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573704.322087:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573704.322090:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516605824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.322097:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573704.322101:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516605824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.322107:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573704.322110:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.322111:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573704.322112:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.322114:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573704.322116:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573704.322119:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573704.322128:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573704.322130:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573704.322135:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516605824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573704.322142:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516605824:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573704.322146:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.322148:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573704.322151:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516605696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.322157:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573704.322159:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573704.322161:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573704.322164:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573704.322165:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573704.322166:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573704.322168:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573704.322170:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788573704.322173:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573704.322174:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573704.322175:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788573704.322177:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573704.322178:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.322180:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573704.322183:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516605824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573726 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573704.322189:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573704.322190:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573704.322191:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573704.322195:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573704.322196:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573704.322198:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573704.322200:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573704.322205:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788573704.322208:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573704.322209:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573704.322211:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788573704.322213:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573704.322214:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.322217:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.322226:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573704.322229:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573704.322231:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573704.322232:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573704.832890:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573704.832895:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573704.832897:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573704.832901:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573704.832908:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573704.832914:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.832920:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573704.832923:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573704.832927:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573705.088914:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309064941, at 4309064960 00000100:00000010:0.0:1788573705.088933:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573705.537008:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573705.537011:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573705.537014:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573705.856913:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573705.856918:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573705.856923:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573705.856927:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573705.856934:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573705.856939:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573705.856961:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573705.856963:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573705.856966:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.369883:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573706.369888:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573706.369892:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573706.369899:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516605952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.369916:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573706.369918:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573706.369923:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573706.369926:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573706.369929:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573706.369934:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516605952:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573706.369939:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573706.369945:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573706.369949:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573706.369952:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573706.369958:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573706.369960:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573706.369967:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573706.369973:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573706.369983:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573706.369991:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516605952, portal 25 00000100:00000001:0.0:1788573706.369995:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573706.369997:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788573706.370004:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516605952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370012:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573706.370016:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573706.370020:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573706.370023:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516605952, offset 0 00000400:00000200:0.0:1788573706.370038:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573706.370049:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573706.370058:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573706.370066:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573706.370071:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573706.370075:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573706.370079:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573706.370090:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546386:546386:256:4294967295] 192.168.202.128@tcp LPNI seq info [546309:546309:8:4294967295] 00000400:00000200:0.0:1788573706.370095:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573706.370108:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573706.370116:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573706.370127:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573706.370136:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573706.370140:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573706.370156:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573706.370159:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573706.370165:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573706.370171:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573706.370178:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573706.370182:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573706.370189:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573706.370194:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516605952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370204:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573706.370208:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516605952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370216:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.370218:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573706.370224:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573706.370229:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573706.370232:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309070062 00000100:00000010:0.0:1788573706.370238:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573706.370247:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573706.370256:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309070062 00000100:00000010:0.0:1788573706.370259:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50ae0. 00000100:00000001:0.0:1788573706.370263:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573706.370267:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573706.370275:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.370278:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573706.370283:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573706/real 1788573706] req@ffff8fc4ac820e00 x1875440516605952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573706.370295:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573706.370297:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573706.370300:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573706.370302:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573706.370305:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573706.370308:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573706.370311:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516605952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573706.370320:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516605952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370326:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573706.370328:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.370331:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573706.370333:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573706.370337:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516605952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370344:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573706.370350:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516605952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370357:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573706.370361:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.370363:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573706.370364:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.370366:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573706.370368:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573706.370371:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573706.370377:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573706.370385:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573706.370390:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516605952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573706.370399:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516605952:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573706.370404:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573706.370408:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516606080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370414:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573706.370416:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573706.370418:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573706.370421:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573706.370423:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573706.370426:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606080:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573706.370430:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573706.370432:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573706.370434:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573706.370435:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573706.370439:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573706.370441:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573706.370445:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573706.370448:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573706.370453:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573706.370456:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516606080, portal 10 00000100:00000001:0.0:1788573706.370459:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573706.370460:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788573706.370467:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516606080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370474:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573706.370477:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573706.370479:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573706.370488:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516606080, offset 0 00000400:00000200:0.0:1788573706.370493:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573706.370498:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573706.370501:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573706.370504:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573706.370508:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573706.370511:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573706.370513:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573706.370516:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546387:546387:256:4294967295] 192.168.202.128@tcp LPNI seq info [546310:546310:8:4294967295] 00000400:00000200:0.0:1788573706.370519:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573706.370526:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573706.370530:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573706.370533:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573706.370538:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573706.370541:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573706.370551:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573706.370553:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573706.370556:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573706.370559:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573706.370562:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573706.370564:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573706.370567:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573706.370571:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516606080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370579:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573706.370583:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516606080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370590:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.370591:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573706.370595:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573706.370606:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573706.370610:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573706.370613:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309066259 00000100:00000001:0.0:1788573706.370616:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309070062 00000100:00000001:0.0:1788573706.370618:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309066259 00000100:00000001:0.0:1788573706.370620:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309070062 00000100:00000001:0.0:1788573706.370623:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573706.370625:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.370627:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573706.370632:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573706/real 1788573706] req@ffff8fc4ac821880 x1875440516606080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573706.370641:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573706.370643:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573706.370645:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573706.370648:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573706.370650:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573706.370653:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573706.370656:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516606080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573706.370662:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516606080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370667:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573706.370668:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.370670:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573706.370671:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573706.370689:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516606080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370696:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573706.370701:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516606080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370715:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573706.370717:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.370719:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573706.370721:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.370723:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573706.370725:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573706.370728:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573706.370732:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573706.370734:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573706.370738:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516606080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573706.370746:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606080:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573706.370751:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573706.370756:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516606208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370763:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573706.370765:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573706.370767:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573706.370769:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573706.370772:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573706.370775:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606208:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573706.370779:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573706.370781:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573706.370782:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573706.370784:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573706.370789:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573706.370791:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573706.370803:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573706.370806:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573706.370865:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573706.370869:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516606208, portal 10 00000100:00000001:0.0:1788573706.370872:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573706.370873:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788573706.370879:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516606208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370887:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573706.370889:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573706.370892:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573706.370894:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516606208, offset 0 00000400:00000200:0.0:1788573706.370899:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573706.370908:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573706.370911:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573706.370914:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573706.370916:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573706.370917:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573706.370919:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573706.370922:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546388:546388:256:4294967295] 192.168.202.128@tcp LPNI seq info [546311:546311:8:4294967295] 00000400:00000200:0.0:1788573706.370925:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573706.370929:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573706.370933:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573706.370935:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573706.370938:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573706.370940:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573706.370948:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573706.370949:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573706.370952:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573706.370954:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573706.370956:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573706.370962:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573706.370965:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573706.370968:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516606208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370974:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573706.370976:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516606208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.370980:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.370982:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573706.370986:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573706.370989:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573706.370992:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573706.370994:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309066267 00000100:00000001:0.0:1788573706.370997:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309070062 00000100:00000001:0.0:1788573706.370999:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309066267 00000100:00000001:0.0:1788573706.371001:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309070062 00000100:00000001:0.0:1788573706.371004:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573706.371006:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.371008:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573706.371012:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573706/real 1788573706] req@ffff8fc4ac820a80 x1875440516606208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573706.371021:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573706.371023:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573706.371025:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573706.371027:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573706.371029:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573706.371032:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573706.371035:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516606208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573706.371043:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516606208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371053:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573706.371054:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371056:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573706.371057:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573706.371059:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516606208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371064:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573706.371067:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516606208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371071:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573706.371073:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371074:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573706.371074:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371076:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573706.371077:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573706.371079:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573706.371081:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573706.371082:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573706.371085:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516606208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573706.371090:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606208:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573706.371093:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573706.371095:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516606336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371100:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573706.371101:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573706.371106:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573706.371107:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573706.371108:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573706.371111:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606336:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573706.371113:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573706.371115:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573706.371116:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573706.371117:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573706.371119:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573706.371121:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573706.371124:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573706.371125:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573706.371128:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573706.371130:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516606336, portal 4 00000100:00000001:0.0:1788573706.371132:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573706.371133:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788573706.371137:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516606336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371141:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573706.371143:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573706.371144:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573706.371146:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516606336, offset 0 00000400:00000200:0.0:1788573706.371148:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573706.371150:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573706.371153:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573706.371155:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573706.371156:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573706.371158:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573706.371159:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573706.371162:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546389:546389:256:4294967295] 192.168.202.128@tcp LPNI seq info [546312:546312:8:4294967295] 00000400:00000200:0.0:1788573706.371165:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573706.371170:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573706.371176:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573706.371178:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573706.371181:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573706.371183:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573706.371190:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573706.371192:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573706.371194:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573706.371196:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573706.371198:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573706.371199:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573706.371201:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573706.371209:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516606336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371215:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573706.371217:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516606336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371221:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.371222:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573706.371224:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573706.371227:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573706.371229:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573706.371231:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309066407 00000100:00000001:0.0:1788573706.371232:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309070062 00000100:00000001:0.0:1788573706.371234:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309066407 00000100:00000001:0.0:1788573706.371235:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309070062 00000100:00000001:0.0:1788573706.371237:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573706.371238:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.371239:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573706.371241:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573706/real 1788573706] req@ffff8fc4ac823800 x1875440516606336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573706.371250:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573706.371251:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573706.371253:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573706.371254:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573706.371255:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573706.371257:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573706.371259:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516606336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573706.371265:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516606336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371269:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573706.371270:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371271:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573706.371272:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573706.371274:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516606336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371278:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573706.371280:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516606336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371284:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573706.371285:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371286:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573706.371287:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371288:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573706.371290:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573706.371291:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573706.371293:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573706.371295:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573706.371300:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516606336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573706.371305:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606336:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573706.371308:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573706.371310:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516606464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371314:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573706.371315:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573706.371316:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573706.371317:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573706.371319:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573706.371320:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606464:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573706.371322:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573706.371323:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573706.371324:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573706.371325:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573706.371327:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573706.371329:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573706.371331:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573706.371333:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573706.371335:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573706.371337:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516606464, portal 4 00000100:00000001:0.0:1788573706.371338:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573706.371339:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788573706.371343:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516606464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371350:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573706.371352:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573706.371354:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573706.371357:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516606464, offset 0 00000400:00000200:0.0:1788573706.371368:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573706.371373:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573706.371376:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573706.371379:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573706.371382:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573706.371384:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573706.371387:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573706.371390:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546390:546390:256:4294967295] 192.168.202.128@tcp LPNI seq info [546313:546313:8:4294967295] 00000400:00000200:0.0:1788573706.371395:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573706.371401:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573706.371405:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573706.371408:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573706.371411:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573706.371413:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573706.371420:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573706.371421:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573706.371423:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573706.371426:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573706.371428:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573706.371429:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573706.371431:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573706.371434:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516606464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371439:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573706.371441:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516606464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573731 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371445:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.371446:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573706.371448:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573706.371451:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573706.371453:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573706.371459:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309066409 00000100:00000001:0.0:1788573706.371460:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309070062 00000100:00000001:0.0:1788573706.371462:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309066409 00000100:00000001:0.0:1788573706.371463:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309070062 00000100:00000001:0.0:1788573706.371465:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573706.371466:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.371468:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573706.371470:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573706/real 1788573706] req@ffff8fc4ac821c00 x1875440516606464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573706.371476:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573706.371477:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573706.371478:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573706.371479:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573706.371480:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573706.371482:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573706.371484:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516606464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573706.371489:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516606464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371493:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573706.371494:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371495:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573706.371496:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573706.371498:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516606464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371502:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573706.371504:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516606464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371509:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573706.371513:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371514:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573706.371515:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371516:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573706.371517:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573706.371519:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573706.371521:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573706.371523:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573706.371525:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516606464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573706.371530:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606464:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573706.371532:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371534:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573706.371536:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516605952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371541:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573706.371543:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573706.371545:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573706.371548:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573706.371549:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573706.371550:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573706.371552:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573706.371554:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788573706.371556:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573706.371558:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573706.371559:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788573706.371562:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573706.371563:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371564:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573706.371566:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516606080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371577:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573706.371578:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573706.371580:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573706.371582:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573706.371582:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573706.371583:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573706.371584:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573706.371585:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788573706.371587:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573706.371587:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573706.371588:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788573706.371589:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573706.371590:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371591:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573706.371593:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516606208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371597:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573706.371598:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573706.371599:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573706.371601:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573706.371601:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573706.371602:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573706.371603:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573706.371604:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788573706.371605:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573706.371606:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573706.371607:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788573706.371608:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573706.371609:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371610:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573706.371611:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516606336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371615:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573706.371616:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573706.371619:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573706.371621:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573706.371622:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573706.371623:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573706.371624:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573706.371625:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788573706.371626:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573706.371627:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573706.371628:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788573706.371629:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573706.371630:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371631:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573706.371633:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516606464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573731 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573706.371637:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573706.371637:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573706.371638:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573706.371639:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573706.371640:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573706.371641:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573706.371642:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573706.371643:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788573706.371644:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573706.371644:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573706.371645:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788573706.371646:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573706.371647:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371648:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.371654:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573706.371655:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.371657:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573706.371658:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.880904:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573706.880908:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573706.880912:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573706.880917:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573706.880923:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573706.880942:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.880949:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573706.880951:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.880954:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573706.880972:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573706.880978:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573706.880991:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573706.881008:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573706.881024:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573706.881028:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.881031:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573706.881033:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573706.881035:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573707.904905:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573707.904913:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573707.904916:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573707.904924:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573707.904931:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573707.904936:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573707.904942:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573707.904945:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573707.904947:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573708.928920:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573708.928927:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573708.928932:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573708.928938:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573708.928945:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573708.928950:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573708.928955:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573708.928960:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573708.928967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573709.952904:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573709.952913:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573709.952918:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573709.952926:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573709.952935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573709.952942:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573709.952959:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573709.952979:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573709.952984:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573710.208914:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309070062, at 4309070080 00000100:00000010:0.0:1788573710.208936:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573710.656999:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573710.657002:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573710.657005:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573710.976923:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573710.976931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573710.976934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573710.976941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573710.976946:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573710.976949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573710.976950:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573710.976952:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573710.976954:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573710.976982:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573710.977108:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573710.977109:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573710.977111:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573710.977113:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573710.977115:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573710.977116:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573710.977117:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573710.977119:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.489076:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573711.489081:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573711.489085:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573711.489096:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516606592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.489117:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573711.489121:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573711.489127:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573711.489131:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573711.489135:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573711.489140:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516606592:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573711.489156:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573711.489161:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573711.489166:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573711.489170:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573711.489176:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573711.489180:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573711.489188:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573711.489193:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573711.489202:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573711.489209:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516606592, portal 25 00000100:00000001:0.0:1788573711.489215:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573711.489219:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788573711.489231:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516606592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.489240:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573711.489247:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573711.489252:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573711.489258:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516606592, offset 0 00000400:00000200:0.0:1788573711.489268:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573711.489278:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573711.489288:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573711.489296:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573711.489304:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573711.489312:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573711.489317:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573711.489325:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546391:546391:256:4294967295] 192.168.202.128@tcp LPNI seq info [546314:546314:8:4294967295] 00000400:00000200:0.0:1788573711.489336:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573711.489348:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573711.489359:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573711.489364:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573711.489374:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573711.489386:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 52 secs later) 00000800:00000100:0.0:1788573711.489412:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573711.489416:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573711.489423:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573711.489431:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573711.489449:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573711.489453:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573711.489460:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573711.489469:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516606592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.489484:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573711.489489:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516606592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.489508:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.489512:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573711.489519:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573711.489528:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573711.489532:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309075181 00000100:00000010:0.0:1788573711.489538:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573711.489566:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573711.489574:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309075181 00000100:00000010:0.0:1788573711.489582:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b502a0. 00000100:00000001:0.0:1788573711.489591:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573711.489598:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573711.489605:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.489610:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573711.489621:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573711/real 1788573711] req@ffff8fc4ac821c00 x1875440516606592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573711.489647:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573711.489650:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573711.489660:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573711.489664:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573711.489668:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573711.489674:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573711.489682:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516606592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573711.489713:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516606592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.489727:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573711.489729:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.489733:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573711.489736:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573711.489745:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516606592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.489766:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573711.489774:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516606592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.489786:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573711.489799:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.489803:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573711.489805:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.489841:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573711.489845:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573711.489850:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573711.489858:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573711.489862:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573711.489871:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516606592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573711.489891:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516606592:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573711.489911:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.489916:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573711.489924:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516606592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.489944:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573711.489949:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573711.489954:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573711.489959:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573711.489963:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573711.489964:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573711.489967:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573711.489970:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788573711.489975:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573711.489977:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573711.489979:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788573711.489982:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573711.489984:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.489987:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.489994:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573711.489996:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.489998:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573711.490000:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573711.490036:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573711.490043:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516606720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490049:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573711.490050:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573711.490053:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573711.490055:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573711.490057:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573711.490060:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573711.490063:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573711.490065:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573711.490066:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573711.490073:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573711.490075:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573711.490077:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573711.490081:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573711.490083:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573711.490086:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573711.490088:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516606720, portal 10 00000100:00000001:0.0:1788573711.490090:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573711.490091:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788573711.490097:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516606720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490102:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573711.490104:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573711.490106:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573711.490107:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516606720, offset 0 00000400:00000200:0.0:1788573711.490110:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573711.490112:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573711.490115:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573711.490117:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573711.490119:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573711.490121:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573711.490122:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573711.490125:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546392:546392:256:4294967295] 192.168.202.128@tcp LPNI seq info [546315:546315:8:4294967295] 00000400:00000200:0.0:1788573711.490128:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573711.490133:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573711.490136:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573711.490138:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573711.490141:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573711.490143:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 52 secs later) 00000800:00000100:0.0:1788573711.490150:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573711.490151:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573711.490157:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573711.490159:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573711.490162:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573711.490163:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573711.490165:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573711.490167:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516606720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490173:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573711.490175:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516606720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490179:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.490180:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573711.490182:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573711.490185:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573711.490187:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573711.490189:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309071379 00000100:00000001:0.0:1788573711.490191:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309075181 00000100:00000001:0.0:1788573711.490192:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309071379 00000100:00000001:0.0:1788573711.490194:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309075181 00000100:00000001:0.0:1788573711.490195:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573711.490196:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.490198:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573711.490200:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573711/real 1788573711] req@ffff8fc4ac823800 x1875440516606720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573711.490205:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573711.490206:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573711.490207:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573711.490209:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573711.490210:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573711.490211:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573711.490216:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516606720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573711.490222:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516606720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490226:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573711.490227:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490228:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573711.490229:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573711.490232:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516606720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490236:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573711.490238:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516606720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490243:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573711.490245:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490246:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573711.490247:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490248:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573711.490249:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573711.490251:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573711.490253:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573711.490254:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573711.490256:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516606720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573711.490261:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573711.490263:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573711.490265:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516606848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490278:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573711.490279:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573711.490280:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573711.490282:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573711.490283:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573711.490285:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606848:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573711.490287:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573711.490288:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573711.490289:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573711.490291:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573711.490293:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573711.490294:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573711.490296:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573711.490298:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573711.490300:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573711.490302:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516606848, portal 10 00000100:00000001:0.0:1788573711.490303:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573711.490304:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788573711.490308:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516606848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490312:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573711.490314:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573711.490315:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573711.490317:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516606848, offset 0 00000400:00000200:0.0:1788573711.490319:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573711.490321:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573711.490324:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573711.490326:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573711.490327:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573711.490329:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573711.490330:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573711.490336:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546393:546393:256:4294967295] 192.168.202.128@tcp LPNI seq info [546316:546316:8:4294967295] 00000400:00000200:0.0:1788573711.490339:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573711.490343:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573711.490347:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573711.490349:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573711.490351:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573711.490353:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 52 secs later) 00000800:00000100:0.0:1788573711.490363:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573711.490364:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573711.490366:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573711.490369:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573711.490371:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573711.490372:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573711.490374:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573711.490376:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516606848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490381:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573711.490383:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516606848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490388:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.490388:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573711.490390:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573711.490392:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573711.490394:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573711.490396:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309071386 00000100:00000001:0.0:1788573711.490397:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309075181 00000100:00000001:0.0:1788573711.490398:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309071386 00000100:00000001:0.0:1788573711.490400:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309075181 00000100:00000001:0.0:1788573711.490401:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573711.490415:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.490416:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573711.490419:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573711/real 1788573711] req@ffff8fc4ac820a80 x1875440516606848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573711.490424:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573711.490425:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573711.490427:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573711.490428:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573711.490429:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573711.490430:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573711.490432:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516606848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573711.490438:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516606848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490442:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573711.490443:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490444:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573711.490445:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573711.490447:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516606848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490451:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573711.490453:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516606848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490457:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573711.490458:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490459:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573711.490460:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490461:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573711.490463:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573711.490464:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573711.490469:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573711.490471:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573711.490474:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516606848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573711.490479:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606848:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573711.490481:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573711.490482:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516606976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490486:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573711.490487:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573711.490488:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573711.490490:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573711.490491:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573711.490493:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573711.490495:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573711.490496:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573711.490497:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573711.490498:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573711.490500:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573711.490501:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573711.490504:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573711.490505:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573711.490507:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573711.490509:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516606976, portal 4 00000100:00000001:0.0:1788573711.490511:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573711.490511:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788573711.490515:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516606976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490519:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573711.490524:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573711.490525:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573711.490527:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516606976, offset 0 00000400:00000200:0.0:1788573711.490530:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573711.490532:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573711.490535:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573711.490537:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573711.490539:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573711.490540:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573711.490542:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573711.490544:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546394:546394:256:4294967295] 192.168.202.128@tcp LPNI seq info [546317:546317:8:4294967295] 00000400:00000200:0.0:1788573711.490560:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573711.490565:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573711.490569:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573711.490571:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573711.490574:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573711.490576:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 52 secs later) 00000800:00000100:0.0:1788573711.490584:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573711.490586:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573711.490589:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573711.490592:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573711.490595:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573711.490598:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573711.490600:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573711.490604:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516606976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490613:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573711.490616:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516606976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490623:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.490625:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573711.490634:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573711.490636:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573711.490639:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573711.490640:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309071526 00000100:00000001:0.0:1788573711.490642:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309075181 00000100:00000001:0.0:1788573711.490643:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309071526 00000100:00000001:0.0:1788573711.490645:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309075181 00000100:00000001:0.0:1788573711.490646:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573711.490648:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.490649:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573711.490652:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573711/real 1788573711] req@ffff8fc4ac821880 x1875440516606976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573711.490656:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573711.490657:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573711.490659:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573711.490660:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573711.490661:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573711.490663:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573711.490665:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516606976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573711.490670:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516606976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490675:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573711.490675:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490677:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573711.490677:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573711.490679:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516606976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490683:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573711.490686:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516606976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490693:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573711.490695:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490696:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573711.490697:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490698:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573711.490699:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573711.490701:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573711.490703:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573711.490704:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573711.490707:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516606976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573711.490712:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516606976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573711.490714:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573711.490716:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516607104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490720:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573711.490721:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573711.490722:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573711.490723:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573711.490725:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573711.490726:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516607104:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573711.490728:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573711.490729:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573711.490730:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573711.490731:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573711.490733:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573711.490735:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573711.490737:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573711.490742:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573711.490744:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573711.490746:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516607104, portal 4 00000100:00000001:0.0:1788573711.490747:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573711.490748:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788573711.490752:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516607104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490756:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573711.490758:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573711.490759:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573711.490761:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516607104, offset 0 00000400:00000200:0.0:1788573711.490763:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573711.490766:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573711.490768:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573711.490770:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573711.490771:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573711.490773:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573711.490775:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573711.490777:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546395:546395:256:4294967295] 192.168.202.128@tcp LPNI seq info [546318:546318:8:4294967295] 00000400:00000200:0.0:1788573711.490779:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573711.490783:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573711.490786:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573711.490788:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573711.490790:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573711.490792:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 52 secs later) 00000800:00000100:0.0:1788573711.490799:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573711.490800:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573711.490802:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573711.490805:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573711.490806:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573711.490825:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573711.490831:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573711.490834:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516607104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490840:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573711.490842:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516607104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573736 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490846:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.490847:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573711.490850:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573711.490851:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573711.490854:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573711.490855:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309071529 00000100:00000001:0.0:1788573711.490857:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309075181 00000100:00000001:0.0:1788573711.490858:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309071529 00000100:00000001:0.0:1788573711.490860:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309075181 00000100:00000001:0.0:1788573711.490861:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573711.490862:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.490864:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573711.490866:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573711/real 1788573711] req@ffff8fc4ac820e00 x1875440516607104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573711.490870:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573711.490871:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573711.490872:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573711.490874:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573711.490875:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573711.490876:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573711.490878:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516607104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573711.490883:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516607104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490894:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573711.490895:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490897:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573711.490898:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573711.490900:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516607104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490905:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573711.490907:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516607104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490911:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573711.490913:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490914:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573711.490914:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490915:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573711.490917:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573711.490918:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573711.490920:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573711.490921:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573711.490924:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516607104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573711.490928:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516607104:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573711.490931:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490932:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573711.490934:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516606720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490938:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573711.490939:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573711.490943:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573711.490945:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573711.490946:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573711.490947:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573711.490950:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573711.490951:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788573711.490954:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573711.490955:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573711.490957:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788573711.490959:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573711.490961:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490963:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573711.490967:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516606848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.490974:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573711.490975:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573711.490977:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573711.490979:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573711.490980:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573711.490982:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573711.490984:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573711.490985:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788573711.490988:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573711.490989:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573711.490990:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788573711.490993:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573711.490995:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.490996:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573711.491000:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516606976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.491006:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573711.491008:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573711.491009:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573711.491011:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573711.491012:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573711.491013:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573711.491018:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573711.491020:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788573711.491022:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573711.491023:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573711.491024:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788573711.491025:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573711.491026:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.491027:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573711.491030:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516607104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573736 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573711.491034:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573711.491035:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573711.491036:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573711.491037:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573711.491038:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573711.491038:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573711.491039:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573711.491040:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788573711.491041:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573711.491042:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573711.491043:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788573711.491044:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573711.491045:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.491046:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573711.491049:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573711.491049:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573711.491051:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573711.491052:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573712.000894:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573712.000901:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573712.000903:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573712.000912:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573712.000919:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573712.000924:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573712.000929:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573712.000933:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573712.000941:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573714.049002:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573714.049011:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573714.049021:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573714.049033:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573714.049046:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573714.049050:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573714.049089:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573714.049094:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573714.049099:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573715.073036:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573715.073054:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573715.073061:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573715.073069:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573715.073078:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573715.073087:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573715.073109:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573715.073115:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573715.073123:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573715.328964:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309075181, at 4309075200 00000100:00000010:0.0:1788573715.328981:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573715.777040:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573715.777045:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573715.777050:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573716.096894:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573716.096903:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573716.096909:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573716.096918:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573716.096926:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573716.096932:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.096940:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573716.096949:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.096961:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.609125:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573716.609132:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573716.609135:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573716.609143:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516607232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.609172:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573716.609175:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573716.609181:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573716.609186:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573716.609190:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573716.609195:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516607232:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573716.609202:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573716.609207:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573716.609211:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573716.609215:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573716.609220:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573716.609224:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573716.609233:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573716.609239:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573716.609247:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573716.609255:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516607232, portal 25 00000100:00000001:0.0:1788573716.609260:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573716.609262:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788573716.609272:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516607232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.609287:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573716.609294:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573716.609298:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573716.609303:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516607232, offset 0 00000400:00000200:0.0:1788573716.609310:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573716.609318:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573716.609326:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573716.609333:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573716.609340:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573716.609345:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573716.609350:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573716.609365:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546396:546396:256:4294967295] 192.168.202.128@tcp LPNI seq info [546319:546319:8:4294967295] 00000400:00000200:0.0:1788573716.609377:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573716.609399:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573716.609409:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573716.609416:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573716.609422:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573716.609438:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788573716.609458:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573716.609463:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573716.609470:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573716.609480:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573716.609487:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573716.609490:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573716.609496:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573716.609503:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516607232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.609520:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573716.609528:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516607232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.609545:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.609550:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573716.609559:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573716.609568:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573716.609574:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309080301 00000100:00000010:0.0:1788573716.609582:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573716.609590:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573716.609599:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309080301 00000100:00000010:0.0:1788573716.609606:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50d80. 00000100:00000001:0.0:1788573716.609611:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573716.609616:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573716.609627:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.609631:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573716.609637:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573716/real 1788573716] req@ffff8fc4ac820700 x1875440516607232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573716.609656:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573716.609660:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573716.609663:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573716.609668:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573716.609670:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573716.609674:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573716.609680:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516607232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573716.609701:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516607232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.609713:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573716.609715:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.609719:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573716.609722:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573716.609728:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516607232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.609735:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573716.609741:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516607232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.609753:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573716.609757:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.609760:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573716.609761:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.609765:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573716.609770:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573716.609775:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573716.609783:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573716.609793:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573716.609800:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516607232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573716.609860:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516607232:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573716.609870:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.609874:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573716.609879:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516607232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.609892:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573716.609897:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573716.609901:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573716.609906:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573716.609909:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573716.609912:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573716.609916:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573716.609920:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788573716.609927:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573716.609931:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573716.609935:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788573716.609940:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573716.609942:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.609945:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.609953:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573716.609955:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.609958:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573716.609969:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573716.610008:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573716.610013:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516607360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610019:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573716.610021:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573716.610023:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573716.610025:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573716.610032:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573716.610035:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516607360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573716.610038:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573716.610040:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573716.610042:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573716.610043:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573716.610045:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573716.610047:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573716.610050:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573716.610052:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573716.610055:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573716.610057:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516607360, portal 10 00000100:00000001:0.0:1788573716.610058:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573716.610059:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788573716.610064:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516607360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610068:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573716.610070:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573716.610071:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573716.610073:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516607360, offset 0 00000400:00000200:0.0:1788573716.610075:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573716.610078:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573716.610081:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573716.610083:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573716.610085:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573716.610087:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573716.610089:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573716.610091:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546397:546397:256:4294967295] 192.168.202.128@tcp LPNI seq info [546320:546320:8:4294967295] 00000400:00000200:0.0:1788573716.610094:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573716.610099:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573716.610102:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573716.610108:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573716.610111:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573716.610113:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788573716.610120:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573716.610122:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573716.610124:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573716.610126:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573716.610129:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573716.610130:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573716.610132:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573716.610135:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516607360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610141:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573716.610143:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516607360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610147:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.610148:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573716.610150:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573716.610153:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573716.610155:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573716.610157:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309076499 00000100:00000001:0.0:1788573716.610159:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309080301 00000100:00000001:0.0:1788573716.610160:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309076499 00000100:00000001:0.0:1788573716.610162:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309080301 00000100:00000001:0.0:1788573716.610163:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573716.610165:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.610166:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573716.610168:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573716/real 1788573716] req@ffff8fc4ac822300 x1875440516607360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573716.610182:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573716.610183:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573716.610185:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573716.610186:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573716.610187:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573716.610189:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573716.610191:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516607360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573716.610197:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516607360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610201:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573716.610202:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610203:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573716.610204:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573716.610206:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516607360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610210:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573716.610213:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516607360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610217:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573716.610218:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610219:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573716.610220:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610221:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573716.610223:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573716.610224:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573716.610226:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573716.610228:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573716.610234:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516607360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573716.610239:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516607360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573716.610242:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573716.610244:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516607488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610247:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573716.610248:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573716.610250:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573716.610251:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573716.610252:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573716.610254:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516607488:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573716.610256:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573716.610257:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573716.610259:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573716.610260:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573716.610262:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573716.610263:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573716.610265:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573716.610267:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573716.610269:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573716.610271:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516607488, portal 10 00000100:00000001:0.0:1788573716.610272:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573716.610273:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788573716.610278:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516607488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610282:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573716.610284:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573716.610285:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573716.610286:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516607488, offset 0 00000400:00000200:0.0:1788573716.610292:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573716.610295:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573716.610297:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573716.610299:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573716.610301:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573716.610303:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573716.610304:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573716.610306:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546398:546398:256:4294967295] 192.168.202.128@tcp LPNI seq info [546321:546321:8:4294967295] 00000400:00000200:0.0:1788573716.610309:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573716.610314:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573716.610316:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573716.610318:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573716.610321:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573716.610322:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788573716.610330:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573716.610331:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573716.610333:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573716.610335:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573716.610337:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573716.610339:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573716.610340:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573716.610343:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516607488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610348:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573716.610350:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516607488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610354:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.610355:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573716.610357:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573716.610359:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573716.610361:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573716.610366:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309076506 00000100:00000001:0.0:1788573716.610368:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309080301 00000100:00000001:0.0:1788573716.610369:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309076506 00000100:00000001:0.0:1788573716.610370:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309080301 00000100:00000001:0.0:1788573716.610372:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573716.610373:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.610374:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573716.610377:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573716/real 1788573716] req@ffff8fc4ac823480 x1875440516607488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573716.610382:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573716.610383:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573716.610384:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573716.610386:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573716.610387:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573716.610388:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573716.610390:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516607488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573716.610396:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516607488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610400:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573716.610400:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610402:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573716.610402:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573716.610405:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516607488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610408:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573716.610411:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516607488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610415:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573716.610419:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610420:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573716.610421:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610422:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573716.610425:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573716.610440:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573716.610443:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573716.610445:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573716.610447:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516607488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573716.610453:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516607488:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573716.610455:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573716.610457:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516607616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610461:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573716.610462:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573716.610464:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573716.610465:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573716.610467:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573716.610468:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516607616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573716.610470:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573716.610471:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573716.610473:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573716.610474:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573716.610476:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573716.610477:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573716.610480:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573716.610482:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573716.610484:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573716.610487:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516607616, portal 4 00000100:00000001:0.0:1788573716.610491:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573716.610492:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788573716.610497:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516607616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610501:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573716.610503:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573716.610504:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573716.610506:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516607616, offset 0 00000400:00000200:0.0:1788573716.610509:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573716.610511:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573716.610514:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573716.610516:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573716.610519:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573716.610521:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573716.610524:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573716.610526:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546399:546399:256:4294967295] 192.168.202.128@tcp LPNI seq info [546322:546322:8:4294967295] 00000400:00000200:0.0:1788573716.610531:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573716.610537:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573716.610541:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573716.610545:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573716.610549:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573716.610552:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788573716.610564:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573716.610566:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573716.610569:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573716.610572:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573716.610575:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573716.610576:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573716.610578:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573716.610581:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516607616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610595:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573716.610597:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516607616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610602:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.610603:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573716.610606:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573716.610608:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573716.610611:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573716.610612:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309076646 00000100:00000001:0.0:1788573716.610614:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309080301 00000100:00000001:0.0:1788573716.610616:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309076646 00000100:00000001:0.0:1788573716.610617:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309080301 00000100:00000001:0.0:1788573716.610619:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573716.610620:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.610621:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573716.610623:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573716/real 1788573716] req@ffff8fc4ac821f80 x1875440516607616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573716.610628:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573716.610629:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573716.610631:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573716.610632:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573716.610633:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573716.610635:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573716.610637:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516607616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573716.610642:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516607616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610646:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573716.610651:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610653:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573716.610654:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573716.610656:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516607616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610661:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573716.610663:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516607616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610668:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573716.610669:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610670:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573716.610671:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610672:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573716.610673:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573716.610675:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573716.610677:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573716.610678:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573716.610680:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516607616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573716.610685:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516607616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573716.610687:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573716.610689:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516607744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610693:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573716.610694:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573716.610696:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573716.610697:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573716.610699:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573716.610703:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516607744:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573716.610706:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573716.610707:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573716.610708:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573716.610709:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573716.610712:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573716.610713:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573716.610715:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573716.610717:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573716.610719:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573716.610721:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516607744, portal 4 00000100:00000001:0.0:1788573716.610722:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573716.610723:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788573716.610727:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516607744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610731:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573716.610733:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573716.610734:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573716.610736:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516607744, offset 0 00000400:00000200:0.0:1788573716.610738:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573716.610740:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573716.610743:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573716.610745:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573716.610746:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573716.610748:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573716.610750:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573716.610752:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546400:546400:256:4294967295] 192.168.202.128@tcp LPNI seq info [546323:546323:8:4294967295] 00000400:00000200:0.0:1788573716.610754:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573716.610758:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573716.610761:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573716.610763:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573716.610769:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573716.610771:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788573716.610778:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573716.610780:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573716.610782:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573716.610784:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573716.610786:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573716.610787:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573716.610789:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573716.610791:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516607744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610797:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573716.610798:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516607744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573741 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610803:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.610804:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573716.610806:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573716.610823:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573716.610826:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573716.610827:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309076649 00000100:00000001:0.0:1788573716.610829:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309080301 00000100:00000001:0.0:1788573716.610830:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309076649 00000100:00000001:0.0:1788573716.610832:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309080301 00000100:00000001:0.0:1788573716.610833:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573716.610835:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.610836:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573716.610839:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573716/real 1788573716] req@ffff8fc4ac822d80 x1875440516607744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573716.610843:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573716.610848:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573716.610849:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573716.610851:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573716.610852:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573716.610854:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573716.610856:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516607744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573716.610862:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516607744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610866:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573716.610867:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610868:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573716.610869:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573716.610872:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516607744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610876:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573716.610878:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516607744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610882:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573716.610883:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610884:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573716.610885:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610887:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573716.610888:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573716.610890:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573716.610892:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573716.610894:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573716.610896:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516607744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573716.610908:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516607744:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573716.610910:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610912:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573716.610914:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516607360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610918:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573716.610919:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573716.610920:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573716.610922:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573716.610924:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573716.610926:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573716.610928:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573716.610930:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788573716.610933:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573716.610934:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573716.610936:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788573716.610939:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573716.610940:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610942:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573716.610946:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516607488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610952:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573716.610954:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573716.610955:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573716.610957:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573716.610959:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573716.610960:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573716.610962:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573716.610964:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788573716.610966:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573716.610968:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573716.610969:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788573716.610978:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573716.610979:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.610981:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573716.610983:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516607616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.610988:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573716.610989:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573716.610990:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573716.610992:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573716.610993:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573716.610994:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573716.610995:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573716.610996:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788573716.610997:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573716.610998:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573716.610999:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788573716.611001:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573716.611001:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.611002:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573716.611004:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516607744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573741 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573716.611008:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573716.611009:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573716.611010:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573716.611011:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573716.611011:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573716.611012:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573716.611013:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573716.611014:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788573716.611015:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573716.611016:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573716.611016:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788573716.611018:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573716.611018:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.611019:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573716.611022:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573716.611026:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573716.611028:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573716.611029:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573718.144970:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573718.144975:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573718.144980:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573718.144987:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573718.144994:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573718.144999:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573718.145002:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573718.145003:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573718.145005:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573719.168960:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573719.168965:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573719.168968:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573719.168971:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573719.168975:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573719.168987:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.168999:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573719.169003:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573719.169008:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573720.192929:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573720.192935:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573720.192938:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573720.192943:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573720.192963:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573720.192968:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573720.192974:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573720.192976:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573720.192979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573720.448875:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309080301, at 4309080320 00000100:00000010:0.0:1788573720.448892:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573720.897021:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573720.897025:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573720.897029:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573721.216890:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573721.216895:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573721.216913:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573721.216917:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573721.216923:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573721.216928:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.216931:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573721.216933:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.216935:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.729263:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573721.729270:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573721.729273:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573721.729281:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5c00 x1875440516607872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.729296:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573721.729299:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573721.729305:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573721.729308:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573721.729312:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573721.729319:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516607872:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573721.729339:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573721.729343:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573721.729345:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573721.729348:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573721.729352:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573721.729354:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573721.729360:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573721.729364:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573721.729371:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573721.729380:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516607872, portal 25 00000100:00000001:0.0:1788573721.729383:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573721.729385:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605327872 : -123400104223744 : ffff8fc4ac9f5c00) 00000100:00000040:0.0:1788573721.729394:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5c00 x1875440516607872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.729409:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573721.729414:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573721.729426:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573721.729431:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516607872, offset 0 00000400:00000200:0.0:1788573721.729438:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573721.729445:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573721.729452:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573721.729459:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573721.729465:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573721.729469:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573721.729474:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573721.729478:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546401:546401:256:4294967295] 192.168.202.128@tcp LPNI seq info [546324:546324:8:4294967295] 00000400:00000200:0.0:1788573721.729485:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573721.729495:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573721.729502:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573721.729507:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573721.729516:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573721.729521:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788573721.729543:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573721.729547:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573721.729554:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573721.729560:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573721.729565:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573721.729568:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573721.729574:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573721.729580:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5c00 x1875440516607872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.729600:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573721.729604:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5c00 x1875440516607872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.729613:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.729616:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573721.729627:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573721.729633:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573721.729635:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309085421 00000100:00000010:0.0:1788573721.729639:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573721.729647:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573721.729654:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309085421 00000100:00000010:0.0:1788573721.729659:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50a20. 00000100:00000001:0.0:1788573721.729663:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573721.729668:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573721.729672:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.729676:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573721.729683:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573721/real 1788573721] req@ffff8fc4ac9f5c00 x1875440516607872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573721.729694:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573721.729696:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573721.729700:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573721.729703:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573721.729706:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573721.729709:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573721.729714:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5c00 x1875440516607872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573721.729725:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5c00 x1875440516607872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.729733:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573721.729737:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.729740:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573721.729743:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573721.729752:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5c00 x1875440516607872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.729763:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573721.729771:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5c00 x1875440516607872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.729786:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573721.729791:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.729795:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573721.729798:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.729802:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573721.729804:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573721.729837:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573721.729845:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573721.729849:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573721.729859:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5c00 x1875440516607872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573721.729879:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516607872:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573721.729885:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.729889:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573721.729895:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5c00 x1875440516607872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.729901:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573721.729903:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573721.729908:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573721.729912:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573721.729915:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573721.729917:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573721.729920:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573721.729925:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788573721.729928:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573721.729933:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573721.729934:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5c00. 00000100:00000001:0.0:1788573721.729936:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573721.729938:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.729941:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.729954:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573721.729957:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.729960:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573721.729962:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573721.729965:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573721.729970:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6d80 x1875440516608000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.729984:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573721.729987:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573721.729999:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573721.730002:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573721.730006:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573721.730012:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516608000:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573721.730018:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573721.730021:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573721.730023:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573721.730027:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573721.730029:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573721.730032:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573721.730038:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573721.730041:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573721.730046:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573721.730050:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516608000, portal 10 00000100:00000001:0.0:1788573721.730053:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573721.730056:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605332352 : -123400104219264 : ffff8fc4ac9f6d80) 00000100:00000040:0.0:1788573721.730065:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6d80 x1875440516608000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730082:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573721.730091:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573721.730096:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573721.730101:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516608000, offset 0 00000400:00000200:0.0:1788573721.730111:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573721.730119:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573721.730133:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573721.730138:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573721.730142:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573721.730145:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573721.730147:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573721.730149:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546402:546402:256:4294967295] 192.168.202.128@tcp LPNI seq info [546325:546325:8:4294967295] 00000400:00000200:0.0:1788573721.730152:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573721.730160:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573721.730167:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573721.730171:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573721.730178:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573721.730182:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788573721.730198:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573721.730201:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573721.730205:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573721.730211:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573721.730214:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573721.730216:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573721.730221:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573721.730226:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6d80 x1875440516608000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730235:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573721.730238:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6d80 x1875440516608000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730246:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.730249:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573721.730254:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573721.730258:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573721.730263:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573721.730267:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309081619 00000100:00000001:0.0:1788573721.730275:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309085421 00000100:00000001:0.0:1788573721.730281:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309081619 00000100:00000001:0.0:1788573721.730284:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309085421 00000100:00000001:0.0:1788573721.730289:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573721.730294:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.730296:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573721.730301:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573721/real 1788573721] req@ffff8fc4ac9f6d80 x1875440516608000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573721.730310:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573721.730313:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573721.730316:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573721.730319:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573721.730320:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573721.730322:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573721.730344:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6d80 x1875440516608000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573721.730360:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6d80 x1875440516608000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730369:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573721.730371:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.730375:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573721.730377:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573721.730383:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6d80 x1875440516608000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730393:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573721.730403:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6d80 x1875440516608000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730418:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573721.730422:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.730431:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573721.730432:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.730437:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573721.730439:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573721.730444:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573721.730449:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573721.730455:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573721.730462:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6d80 x1875440516608000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573721.730474:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516608000:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573721.730478:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.730481:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573721.730486:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6d80 x1875440516608000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730495:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573721.730498:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573721.730501:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573721.730503:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573721.730506:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573721.730510:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573721.730514:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573721.730517:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37c00. 02000000:00000001:0.0:1788573721.730522:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573721.730523:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573721.730526:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6d80. 00000100:00000001:0.0:1788573721.730530:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573721.730533:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.730538:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.730543:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573721.730548:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.730550:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573721.730552:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573721.730560:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573721.730566:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5500 x1875440516608128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730576:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573721.730579:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573721.730585:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573721.730589:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573721.730592:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573721.730598:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516608128:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573721.730603:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573721.730605:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573721.730608:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573721.730612:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573721.730615:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573721.730618:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573721.730625:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573721.730629:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573721.730632:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573721.730636:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516608128, portal 10 00000100:00000001:0.0:1788573721.730641:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573721.730643:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605326080 : -123400104225536 : ffff8fc4ac9f5500) 00000100:00000040:0.0:1788573721.730651:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5500 x1875440516608128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730657:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573721.730661:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573721.730664:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573721.730669:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516608128, offset 0 00000400:00000200:0.0:1788573721.730672:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573721.730675:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573721.730678:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573721.730682:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573721.730686:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573721.730689:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573721.730700:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573721.730703:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546403:546403:256:4294967295] 192.168.202.128@tcp LPNI seq info [546326:546326:8:4294967295] 00000400:00000200:0.0:1788573721.730708:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573721.730716:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573721.730723:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573721.730729:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573721.730734:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573721.730738:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788573721.730753:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573721.730755:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573721.730759:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573721.730763:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573721.730767:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573721.730770:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573721.730773:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573721.730779:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5500 x1875440516608128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730794:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573721.730798:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5500 x1875440516608128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730825:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.730828:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573721.730832:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573721.730836:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573721.730843:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573721.730848:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309081627 00000100:00000001:0.0:1788573721.730854:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309085421 00000100:00000001:0.0:1788573721.730859:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309081627 00000100:00000001:0.0:1788573721.730864:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309085421 00000100:00000001:0.0:1788573721.730875:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573721.730878:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.730880:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573721.730885:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573721/real 1788573721] req@ffff8fc4ac9f5500 x1875440516608128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573721.730892:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573721.730893:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573721.730895:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573721.730897:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573721.730898:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573721.730900:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573721.730902:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5500 x1875440516608128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573721.730908:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5500 x1875440516608128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730912:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573721.730913:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.730914:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573721.730915:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573721.730917:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5500 x1875440516608128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730921:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573721.730924:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5500 x1875440516608128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730928:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573721.730930:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.730931:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573721.730931:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.730933:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573721.730935:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573721.730940:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573721.730942:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573721.730944:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573721.730946:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5500 x1875440516608128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573721.730951:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516608128:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573721.730954:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573721.730956:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4e00 x1875440516608256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.730960:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573721.730961:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573721.730963:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573721.730964:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573721.730965:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573721.730967:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516608256:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573721.730969:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573721.730970:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573721.730971:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573721.730972:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573721.730975:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573721.730976:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573721.730979:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573721.730981:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573721.730983:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573721.730985:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516608256, portal 4 00000100:00000001:0.0:1788573721.730986:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573721.730987:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605324288 : -123400104227328 : ffff8fc4ac9f4e00) 00000100:00000040:0.0:1788573721.730991:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4e00 x1875440516608256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731002:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573721.731004:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573721.731006:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573721.731008:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516608256, offset 0 00000400:00000200:0.0:1788573721.731010:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573721.731013:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573721.731016:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573721.731018:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573721.731019:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573721.731021:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573721.731023:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573721.731025:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546404:546404:256:4294967295] 192.168.202.128@tcp LPNI seq info [546327:546327:8:4294967295] 00000400:00000200:0.0:1788573721.731027:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573721.731032:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573721.731035:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573721.731037:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573721.731040:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573721.731042:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788573721.731049:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573721.731050:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573721.731052:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573721.731055:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573721.731057:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573721.731058:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573721.731060:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573721.731062:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4e00 x1875440516608256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731068:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573721.731070:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4e00 x1875440516608256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731084:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.731086:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573721.731090:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573721.731093:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573721.731097:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573721.731099:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309081767 00000100:00000001:0.0:1788573721.731102:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309085421 00000100:00000001:0.0:1788573721.731104:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309081767 00000100:00000001:0.0:1788573721.731106:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309085421 00000100:00000001:0.0:1788573721.731108:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573721.731111:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.731113:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573721.731118:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573721/real 1788573721] req@ffff8fc4ac9f4e00 x1875440516608256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573721.731126:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573721.731127:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573721.731129:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573721.731131:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573721.731132:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573721.731134:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573721.731136:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4e00 x1875440516608256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573721.731142:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4e00 x1875440516608256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731146:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573721.731147:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731148:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573721.731149:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573721.731152:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4e00 x1875440516608256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731162:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573721.731165:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4e00 x1875440516608256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731170:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573721.731171:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731172:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573721.731173:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731174:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573721.731176:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573721.731177:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573721.731179:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573721.731180:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573721.731183:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4e00 x1875440516608256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573721.731187:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516608256:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573721.731190:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731191:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573721.731193:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5500 x1875440516608128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731197:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573721.731198:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573721.731200:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573721.731201:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573721.731202:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573721.731203:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573721.731204:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573721.731206:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34c00. 02000000:00000001:0.0:1788573721.731210:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573721.731211:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573721.731212:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5500. 00000100:00000001:0.0:1788573721.731214:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573721.731215:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731216:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573721.731218:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4e00 x1875440516608256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731222:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573721.731223:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573721.731224:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573721.731225:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573721.731226:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573721.731226:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573721.731228:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573721.731228:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36400. 02000000:00000001:0.0:1788573721.731230:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573721.731230:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573721.731231:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4e00. 00000100:00000001:0.0:1788573721.731233:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573721.731234:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731235:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731237:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573721.731238:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.731240:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573721.731241:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573721.731242:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573721.731244:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516608384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731249:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573721.731250:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573721.731251:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573721.731253:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573721.731254:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573721.731256:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516608384:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573721.731262:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573721.731263:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573721.731264:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573721.731265:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573721.731267:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573721.731268:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573721.731271:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573721.731273:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573721.731275:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573721.731277:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516608384, portal 4 00000100:00000001:0.0:1788573721.731279:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573721.731279:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788573721.731283:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516608384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731287:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573721.731289:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573721.731291:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573721.731292:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516608384, offset 0 00000400:00000200:0.0:1788573721.731295:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573721.731297:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573721.731300:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573721.731302:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573721.731303:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573721.731305:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573721.731307:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573721.731309:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546405:546405:256:4294967295] 192.168.202.128@tcp LPNI seq info [546328:546328:8:4294967295] 00000400:00000200:0.0:1788573721.731311:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573721.731316:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573721.731318:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573721.731320:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573721.731323:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573721.731342:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788573721.731353:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573721.731355:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573721.731357:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573721.731359:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573721.731362:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573721.731363:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573721.731365:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573721.731367:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516608384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731373:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573721.731375:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516608384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573746 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731379:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.731380:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573721.731382:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573721.731385:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573721.731387:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573721.731388:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309081769 00000100:00000001:0.0:1788573721.731390:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309085421 00000100:00000001:0.0:1788573721.731392:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309081769 00000100:00000001:0.0:1788573721.731393:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309085421 00000100:00000001:0.0:1788573721.731395:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573721.731396:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.731397:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573721.731399:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573721/real 1788573721] req@ffff8fc4ac9f6a00 x1875440516608384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573721.731404:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573721.731405:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573721.731409:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573721.731411:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573721.731412:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573721.731413:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573721.731416:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516608384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573721.731421:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516608384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731425:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573721.731426:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731427:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573721.731428:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573721.731430:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516608384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731434:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573721.731437:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516608384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731441:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573721.731443:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731444:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573721.731445:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731446:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573721.731447:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573721.731449:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573721.731451:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573721.731452:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573721.731454:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516608384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573721.731459:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516608384:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573721.731464:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731465:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573721.731468:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516608384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573746 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573721.731472:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573721.731473:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573721.731474:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573721.731475:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573721.731476:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573721.731477:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573721.731478:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573721.731479:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35c00. 02000000:00000001:0.0:1788573721.731481:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573721.731482:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573721.731483:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788573721.731484:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573721.731485:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731486:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573721.731489:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573721.731490:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573721.731491:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573721.731492:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573722.240894:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573722.240901:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573722.240903:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573722.240910:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573722.240914:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573722.240921:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573722.240926:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573722.240930:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573722.240934:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573723.264929:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573723.264938:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573723.264941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573723.264948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573723.264953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573723.264969:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573723.264974:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573723.264981:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573723.264988:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573724.288971:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573724.288976:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573724.288980:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573724.288985:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573724.288988:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573724.288992:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.288998:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573724.289001:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573724.289008:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573725.312981:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573725.312989:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573725.312993:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573725.313006:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573725.313010:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573725.313013:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.313041:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573725.313045:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573725.313049:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573725.568952:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309085421, at 4309085440 00000100:00000010:0.0:1788573725.568971:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573726.017024:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573726.017027:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573726.017030:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573726.336918:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573726.336923:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573726.336926:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573726.336929:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573726.336933:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573726.336939:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573726.336955:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573726.336961:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573726.336968:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573726.848943:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573726.848988:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573726.848992:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573726.848997:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573726.849003:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573726.849010:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14414 00000400:00000200:0.0:1788573726.849022:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573726.849031:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573726.849036:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573726.849040:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849045:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573726.849048:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849052:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573726.849059:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573726.849063:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573726.849067:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573726.849071:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573726.849075:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573726.849079:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573726.849083:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849087:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573726.849090:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849103:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573726.849107:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573726.849112:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849116:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573726.849121:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573726.849126:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573726.849131:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.849134:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573726.849137:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573726.849147:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573726.849161:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573726.849191:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573726.849196:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.849201:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573726.849205:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573726.849213:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573726.849239:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573726.849251:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573726.849265:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516608512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573726.849292:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573726.849298:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573726.849303:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516608512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573726.849326:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573726.849329:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573726.849333:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573726.849334:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573726.849336:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573726.849339:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573726.849342:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14414 00000400:00000200:0.0:1788573726.849347:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573726.849351:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573726.849353:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573726.849357:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849362:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573726.849364:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849369:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573726.849374:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573726.849375:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573726.849378:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573726.849389:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573726.849395:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573726.849399:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573726.849406:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849409:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573726.849413:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849418:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573726.849423:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573726.849429:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849436:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573726.849439:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573726.849446:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573726.849455:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.849459:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573726.849463:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573726.849470:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.849477:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573726.849481:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573726.849484:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573726.849488:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573726.849490:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.849496:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573726.849498:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573726.849499:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573726.849504:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573726.849510:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573726.849518:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516608640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573726.849526:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573726.849529:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573726.849535:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516608640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573726.849580:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573726.849588:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573726.849597:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573726.849602:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573726.849605:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573726.849614:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573726.849622:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14414 00000400:00000200:0.0:1788573726.849634:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573726.849646:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573726.849653:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573726.849663:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849668:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573726.849676:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849683:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573726.849689:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573726.849694:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573726.849700:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573726.849705:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573726.849709:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573726.849713:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573726.849718:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849735:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573726.849743:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849751:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573726.849758:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573726.849766:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573726.849776:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573726.849788:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573726.849795:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573726.849804:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.849857:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573726.849862:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573726.849867:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.849873:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573726.849883:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573726.849890:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573726.849896:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573726.849898:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.849904:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573726.849907:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573726.849912:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573726.849923:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573726.849933:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573726.849948:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516608768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573726.849970:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573726.849977:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573726.849991:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516608768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573726.850036:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573726.850040:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573726.850051:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573726.850055:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573726.850059:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573726.850063:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573726.850070:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14414 00000400:00000200:0.0:1788573726.850081:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573726.850088:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573726.850090:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573726.850103:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.850108:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573726.850114:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.850119:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573726.850125:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573726.850128:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573726.850133:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573726.850137:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573726.850143:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573726.850150:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573726.850158:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.850162:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573726.850168:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.850170:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573726.850172:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573726.850174:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573726.850175:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573726.850177:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573726.850181:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573726.850187:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.850194:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573726.850198:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573726.850206:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.850210:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573726.850217:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573726.850221:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573726.850243:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573726.850246:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.850251:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573726.850255:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573726.850258:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573726.850266:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573726.850281:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573726.850291:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516608896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573726.850303:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573726.850306:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573726.850310:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516608896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573726.850326:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573726.850331:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573726.850337:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573726.850338:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573726.850339:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573726.850342:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573726.850347:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14414 00000400:00000200:0.0:1788573726.850356:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573726.850364:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573726.850368:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573726.850376:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.850382:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573726.850387:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.850392:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573726.850397:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573726.850400:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573726.850404:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573726.850409:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573726.850415:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573726.850419:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573726.850428:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.850432:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573726.850446:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573726.850449:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573726.850455:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573726.850462:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573726.850466:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573726.850471:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573726.850474:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573726.850478:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.850482:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573726.850486:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573726.850489:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.850492:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573726.850501:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573726.850505:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573726.850512:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573726.850515:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573726.850517:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573726.850518:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573726.850521:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573726.850528:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573726.850536:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573726.850548:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516609024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573726.850574:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573726.850582:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573726.850603:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516609024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573726.850626:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573726.850633:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14424) 00000100:00000001:0.0:1788573727.360974:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573727.360997:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573727.361003:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573727.361037:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573727.361052:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573727.361056:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573727.361072:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573727.361077:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573727.361082:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573728.384986:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573728.384996:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573728.385003:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573728.385020:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573728.385031:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573728.385041:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573728.385052:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573728.385057:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573728.385061:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573729.408904:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573729.408913:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573729.408917:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573729.408924:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573729.408931:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573729.408936:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573729.408947:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573729.408950:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573729.408953:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573729.408985:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573729.408991:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573729.408995:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573729.409001:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573729.409007:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573729.409009:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573729.409011:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573729.409012:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573729.409014:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573730.688873:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309090542, at 4309090560 00000100:00000010:0.0:1788573730.688890:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7de0. 00000020:00000001:0.0:1788573731.136997:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573731.137000:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573731.137018:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573731.456909:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573731.456916:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573731.456919:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573731.456928:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573731.456935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573731.456938:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573731.456942:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573731.456952:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573731.456958:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573731.456989:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573731.456995:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573731.457000:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573731.457005:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573731.457011:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573731.457014:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573731.457019:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573731.457022:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573731.457025:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573732.480941:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573732.480952:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573732.480956:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573732.480963:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573732.480974:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573732.480980:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573732.480991:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573732.480997:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573732.481006:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573733.184979:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573733.184983:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573733.184986:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573733.504917:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573733.504923:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573733.504927:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573733.504932:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573733.504937:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573733.504942:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573733.504957:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573733.504983:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573733.504986:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573733.505021:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573733.505025:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573733.505029:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573733.505032:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573733.505036:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573733.505039:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573733.505043:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573733.505046:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573733.505049:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573734.528925:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573734.528933:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573734.528935:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573734.528941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573734.528946:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573734.528950:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573734.528954:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573734.528958:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573734.528964:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573734.528982:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573734.528988:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573734.528992:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573734.528995:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573734.529001:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573734.529006:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573734.529011:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573734.529014:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573734.529017:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573735.552895:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573735.552902:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573735.552906:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573735.552914:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573735.552920:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573735.552924:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.552932:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573735.552938:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573735.552943:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573735.808900:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309095661, at 4309095680 00000100:00000010:0.0:1788573735.808918:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7d80. 00000020:00000001:0.0:1788573736.256985:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573736.256990:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573736.256993:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573736.576942:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573736.576950:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573736.576953:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573736.576960:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573736.576968:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573736.576974:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573736.576979:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573736.576984:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573736.576991:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573737.088892:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573737.088903:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573737.088905:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573737.088909:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573737.088914:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573737.088922:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14424 00000400:00000200:0.0:1788573737.088932:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573737.088939:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573737.088943:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573737.088946:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.088949:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573737.088952:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.088954:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573737.088958:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573737.088960:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573737.088963:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573737.088966:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573737.088969:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573737.088972:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573737.088995:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.088997:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573737.088999:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089001:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573737.089003:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573737.089006:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089007:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573737.089010:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573737.089013:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573737.089015:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089016:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573737.089018:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573737.089038:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573737.089041:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573737.089044:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573737.089045:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089047:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573737.089050:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573737.089054:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573737.089060:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573737.089066:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573737.089073:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516609792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573737.089081:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573737.089085:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573737.089089:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516609792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573737.089106:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573737.089109:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573737.089113:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573737.089115:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573737.089123:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573737.089126:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573737.089129:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14424 00000400:00000200:0.0:1788573737.089134:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573737.089137:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573737.089140:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573737.089143:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089145:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573737.089147:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089149:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573737.089152:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573737.089153:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573737.089155:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573737.089158:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573737.089160:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573737.089162:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573737.089165:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089167:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573737.089170:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089172:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573737.089174:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573737.089177:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089179:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573737.089181:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573737.089187:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573737.089189:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089191:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573737.089195:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573737.089197:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089199:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573737.089207:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573737.089209:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573737.089212:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573737.089213:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089215:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573737.089217:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573737.089219:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573737.089223:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573737.089227:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573737.089232:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516609920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573737.089240:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573737.089242:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573737.089245:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516609920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573737.089254:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573737.089257:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573737.089260:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573737.089261:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573737.089263:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573737.089266:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573737.089268:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14424 00000400:00000200:0.0:1788573737.089272:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573737.089275:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573737.089277:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573737.089279:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089282:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573737.089284:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089286:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573737.089289:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573737.089293:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573737.089295:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573737.089297:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573737.089299:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573737.089301:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573737.089305:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089307:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573737.089309:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089311:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573737.089314:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573737.089317:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089319:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573737.089321:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573737.089324:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573737.089326:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089328:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573737.089330:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573737.089332:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089334:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573737.089337:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573737.089340:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573737.089342:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573737.089344:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089346:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573737.089347:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573737.089349:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573737.089353:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573737.089356:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573737.089360:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516610048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573737.089368:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573737.089370:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573737.089378:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516610048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573737.089386:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573737.089389:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573737.089393:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573737.089395:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573737.089397:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573737.089399:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573737.089401:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14424 00000400:00000200:0.0:1788573737.089405:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573737.089408:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573737.089410:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573737.089413:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089415:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573737.089417:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089420:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573737.089422:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573737.089423:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573737.089425:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573737.089427:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573737.089429:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573737.089431:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573737.089434:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089436:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573737.089438:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089440:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573737.089443:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573737.089445:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089447:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573737.089455:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573737.089458:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573737.089460:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089462:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573737.089464:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573737.089467:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089469:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573737.089472:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573737.089475:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573737.089477:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573737.089479:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089481:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573737.089482:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573737.089484:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573737.089488:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573737.089491:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573737.089495:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516610176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573737.089503:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573737.089505:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573737.089510:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516610176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573737.089517:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573737.089520:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573737.089523:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573737.089525:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573737.089527:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573737.089529:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573737.089532:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14424 00000400:00000200:0.0:1788573737.089536:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573737.089538:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573737.089541:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573737.089547:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089549:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573737.089551:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089554:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573737.089556:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573737.089557:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573737.089559:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573737.089561:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573737.089563:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573737.089566:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573737.089568:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089570:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573737.089572:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089574:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573737.089583:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573737.089586:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573737.089588:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573737.089589:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573737.089592:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573737.089594:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089596:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573737.089598:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573737.089601:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089602:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573737.089606:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573737.089608:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573737.089611:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573737.089612:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.089614:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573737.089615:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573737.089617:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573737.089620:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573737.089628:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573737.089632:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516610304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573737.089641:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573737.089643:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573737.089647:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516610304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573737.089656:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573737.089661:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14434) 00000100:00000001:0.0:1788573737.600878:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573737.600886:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573737.600888:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573737.600893:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573737.600897:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573737.600900:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573737.600903:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573737.600905:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573737.600911:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573737.600941:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573737.600946:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573737.600950:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573737.600955:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573737.600963:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573737.600968:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573737.600983:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573737.600987:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573737.600990:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573738.305052:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573738.305058:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573738.305062:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573738.624906:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573738.624910:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573738.624913:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573738.624916:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573738.624919:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573738.624941:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573738.624947:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573738.624953:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573738.624959:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573739.648918:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573739.648929:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573739.648934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573739.648943:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573739.648952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573739.648959:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573739.649012:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573739.649017:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573739.649021:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573740.673044:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573740.673051:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573740.673057:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573740.673065:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573740.673072:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573740.673076:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.673080:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573740.673082:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573740.673086:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573740.928923:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309100781, at 4309100800 00000100:00000010:0.0:1788573740.928942:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c00. 00000020:00000001:0.0:1788573741.377070:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573741.377073:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573741.377077:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573741.696909:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573741.696915:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573741.696920:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573741.696929:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573741.696949:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573741.696953:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573741.696977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573741.696982:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573741.696985:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573742.208941:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573742.208969:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573742.208972:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573742.208975:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573742.208978:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573742.208984:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14429 00000400:00000200:0.0:1788573742.208992:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573742.208997:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573742.209000:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573742.209002:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209004:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573742.209006:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209008:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573742.209011:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573742.209013:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573742.209015:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573742.209017:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573742.209018:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573742.209020:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573742.209022:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209023:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573742.209025:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209026:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573742.209028:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573742.209031:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209033:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573742.209035:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573742.209037:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573742.209039:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209041:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573742.209042:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573742.209048:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573742.209053:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573742.209057:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573742.209058:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209060:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573742.209062:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573742.209066:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573742.209073:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573742.209079:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573742.209086:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516610432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573742.209093:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573742.209097:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573742.209100:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516610432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573742.209119:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573742.209121:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573742.209124:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573742.209125:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573742.209126:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573742.209128:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573742.209130:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14429 00000400:00000200:0.0:1788573742.209134:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573742.209136:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573742.209138:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573742.209141:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209145:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573742.209147:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209151:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573742.209155:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573742.209156:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573742.209158:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573742.209166:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573742.209171:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573742.209174:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573742.209179:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209185:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573742.209187:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209188:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573742.209190:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573742.209191:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209196:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573742.209199:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573742.209205:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573742.209209:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209212:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573742.209216:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573742.209220:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209224:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573742.209229:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573742.209232:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573742.209235:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573742.209237:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209239:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573742.209242:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573742.209244:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573742.209252:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573742.209255:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573742.209262:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516610560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573742.209274:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573742.209278:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573742.209286:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516610560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573742.209318:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573742.209323:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573742.209326:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573742.209328:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573742.209331:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573742.209334:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573742.209339:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14429 00000400:00000200:0.0:1788573742.209345:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573742.209351:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573742.209355:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573742.209362:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209366:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573742.209371:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209374:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573742.209377:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573742.209380:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573742.209382:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573742.209384:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573742.209389:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573742.209393:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573742.209397:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209399:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573742.209402:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209405:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573742.209407:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573742.209412:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209414:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573742.209422:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573742.209425:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573742.209431:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209436:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573742.209438:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573742.209443:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209447:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573742.209452:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573742.209455:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573742.209458:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573742.209460:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209463:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573742.209466:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573742.209470:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573742.209474:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573742.209479:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573742.209485:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516610688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573742.209504:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573742.209508:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573742.209513:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516610688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573742.209531:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573742.209536:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573742.209541:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573742.209543:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573742.209545:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573742.209551:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573742.209556:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14429 00000400:00000200:0.0:1788573742.209562:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573742.209566:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573742.209570:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573742.209578:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209581:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573742.209585:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209588:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573742.209591:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573742.209593:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573742.209596:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573742.209600:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573742.209603:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573742.209607:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573742.209612:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209617:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573742.209620:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209625:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573742.209628:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573742.209630:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209632:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573742.209633:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573742.209635:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573742.209636:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209638:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573742.209639:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573742.209641:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209642:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573742.209644:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573742.209648:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573742.209652:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573742.209654:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209658:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573742.209660:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573742.209662:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573742.209665:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573742.209676:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573742.209681:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516610816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573742.209691:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573742.209694:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573742.209700:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516610816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573742.209714:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573742.209718:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573742.209725:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573742.209727:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573742.209731:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573742.209735:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573742.209741:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14429 00000400:00000200:0.0:1788573742.209747:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573742.209752:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573742.209757:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573742.209763:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209767:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573742.209772:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209774:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573742.209779:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573742.209782:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573742.209785:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573742.209788:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573742.209792:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573742.209797:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573742.209802:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209805:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573742.209838:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209842:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573742.209844:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573742.209847:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573742.209849:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573742.209852:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573742.209855:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573742.209859:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209863:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573742.209865:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573742.209868:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209870:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573742.209874:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573742.209878:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573742.209882:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573742.209884:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573742.209887:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573742.209889:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573742.209892:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573742.209899:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573742.209904:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573742.209911:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516610944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573742.209937:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573742.209943:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573742.209953:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516610944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573742.209967:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573742.209972:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14439) 00000100:00000001:0.0:1788573742.720941:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573742.720948:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573742.720961:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573742.720977:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573742.720982:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573742.720990:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573742.720997:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573742.721002:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573742.721007:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573743.425014:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573743.425018:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573743.425021:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573743.744885:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573743.744904:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573743.744907:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573743.744917:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573743.744924:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573743.744929:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573743.744932:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573743.744936:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573743.744942:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573744.768941:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573744.768949:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573744.768951:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573744.768968:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573744.768975:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573744.768979:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573744.768991:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573744.768995:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573744.768999:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573744.769027:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573744.769030:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573744.769031:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573744.769034:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573744.769036:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573744.769038:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573744.769040:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573744.769042:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573744.769046:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573745.792909:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573745.792915:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573745.792936:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573745.792942:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573745.792949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573745.792956:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.792962:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573745.792968:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573745.792974:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573746.048907:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309105901, at 4309105920 00000100:00000010:0.0:1788573746.048926:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ea0. 00000020:00000001:0.0:1788573746.497053:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573746.497058:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573746.497062:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573746.816938:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573746.816947:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573746.816953:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573746.816960:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573746.816964:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573746.816968:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.816973:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573746.816979:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.816985:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573746.817006:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573746.817011:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573746.817016:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573746.817023:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573746.817029:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573746.817032:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.817035:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573746.817037:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573746.817039:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573747.840962:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573747.840973:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573747.840979:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573747.840991:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573747.841003:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573747.841013:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573747.841023:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573747.841044:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573747.841047:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573747.841083:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573747.841087:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573747.841088:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573747.841091:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573747.841094:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573747.841096:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573747.841098:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573747.841100:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573747.841113:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573748.545022:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573748.545028:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573748.545032:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573748.864926:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573748.864936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573748.864941:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573748.864949:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573748.864957:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573748.864964:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573748.864971:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573748.864976:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573748.864983:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573748.865004:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573748.865010:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573748.865014:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573748.865017:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573748.865021:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573748.865025:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573748.865028:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573748.865030:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573748.865034:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573749.888922:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573749.888931:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573749.888934:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573749.888942:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573749.888962:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573749.888968:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573749.888984:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573749.888990:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573749.888995:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573749.889013:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573749.889016:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573749.889017:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573749.889019:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573749.889022:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573749.889024:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573749.889026:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573749.889027:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573749.889030:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573750.912901:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573750.912908:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573750.912912:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573750.912922:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573750.912928:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573750.912934:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573750.912939:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573750.912943:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573750.912954:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573750.912983:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573750.912988:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573750.912990:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573750.912994:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573750.912997:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573750.912999:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573750.913002:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573750.913004:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573750.913007:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573751.168914:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309111022, at 4309111040 00000100:00000010:0.0:1788573751.168929:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7360. 00000020:00000001:0.0:1788573751.617050:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573751.617054:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573751.617057:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573751.936906:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573751.936914:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573751.936932:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573751.936939:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573751.936951:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573751.936954:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573751.936966:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573751.936970:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573751.936972:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.960902:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573752.960909:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573752.960913:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573752.960922:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573752.960930:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573752.960937:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.960944:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573752.960948:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.960953:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573752.960986:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573752.960990:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573752.960991:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573752.960994:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573752.960997:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573752.960999:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.961001:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573752.961002:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573752.961004:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573753.664986:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573753.664990:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573753.664993:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573753.984915:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573753.984923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573753.984927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573753.984933:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573753.984940:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573753.984946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573753.984955:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573753.984960:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573753.984963:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573753.984986:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573753.984998:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573753.985000:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573753.985003:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573753.985006:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573753.985008:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573753.985010:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573753.985012:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573753.985014:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573755.008902:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573755.008907:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573755.008912:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573755.008922:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573755.008929:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573755.008934:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573755.008950:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573755.008953:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573755.008955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573756.032910:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573756.032920:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573756.032924:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573756.032932:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573756.032941:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573756.032947:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573756.033107:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573756.033109:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573756.033111:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573756.288932:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309116141, at 4309116160 00000100:00000010:0.0:1788573756.288949:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ba0. 00000020:00000001:0.0:1788573756.737062:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573756.737065:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573756.737067:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573757.056929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573757.056934:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573757.056935:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573757.056938:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573757.056940:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573757.056942:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573757.056959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573757.056963:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573757.056967:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573758.080914:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573758.080923:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573758.080926:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573758.080936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573758.080945:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573758.080952:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573758.080960:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573758.080965:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573758.080973:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573758.785040:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573758.785044:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573758.785048:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573759.104916:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573759.104921:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573759.104926:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573759.104933:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573759.104944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573759.104947:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573759.104970:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573759.104973:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573759.104977:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573760.128936:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573760.128941:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573760.128943:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573760.128946:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573760.128949:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573760.128952:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.128956:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573760.128960:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.128965:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573760.129015:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573760.129022:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573760.129030:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573760.129042:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573760.129056:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573760.129084:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.129088:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573760.129089:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573760.129092:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573761.152926:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573761.152934:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573761.152937:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573761.152945:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573761.152950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573761.152955:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573761.152960:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573761.152965:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573761.152971:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573761.408923:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309121261, at 4309121280 00000100:00000010:0.0:1788573761.408946:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7420. 00000020:00000001:0.0:1788573761.857001:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573761.857004:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573761.857007:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573762.176929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573762.176936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573762.176942:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573762.176948:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573762.176954:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573762.176960:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.176977:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573762.176980:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573762.176984:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573763.200886:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573763.200893:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573763.200895:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573763.200903:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573763.200908:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573763.200912:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573763.200916:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573763.200920:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573763.200924:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573763.904970:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573763.904988:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573763.904991:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573764.224902:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573764.224911:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573764.224915:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573764.224922:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573764.224929:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573764.224935:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573764.224941:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573764.224946:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573764.224952:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788573764.737125:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573764.737141:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573764.737145:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573764.737147:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573765.248912:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573765.248918:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573765.248922:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573765.248928:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573765.248933:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573765.248938:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573765.248947:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573765.248952:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573765.248957:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573766.272921:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573766.272931:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573766.272936:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573766.272943:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573766.272950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573766.272955:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573766.272966:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573766.272972:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573766.272978:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573766.528902:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309126381, at 4309126400 00000100:00000010:0.0:1788573766.528920:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c60. 00000020:00000001:0.0:1788573766.977057:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573766.977062:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573766.977088:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573767.296874:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573767.296882:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573767.296883:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573767.296888:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573767.296892:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573767.296895:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573767.296897:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573767.296899:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573767.296902:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573767.296941:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573767.296945:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573767.296946:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573767.296948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573767.296951:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573767.296956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573767.296959:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573767.296960:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573767.296962:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573768.321011:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573768.321020:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573768.321023:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573768.321028:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573768.321040:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573768.321051:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573768.321063:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573768.321067:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573768.321078:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573769.025034:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573769.025039:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573769.025043:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573769.344913:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573769.344920:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573769.344923:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573769.344930:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573769.344936:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573769.344940:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573769.344944:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573769.344965:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573769.344968:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573770.369023:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573770.369030:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573770.369035:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573770.369045:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573770.369052:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573770.369059:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573770.369066:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573770.369072:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573770.369080:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788573770.881120:0:2388:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788573770.881160:0:2388:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788573770.881168:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573770.881171:0:2388:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788573770.881175:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573770.881178:0:2388:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788573770.881182:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573770.881185:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793600. 00000400:00000200:0.0:1788573770.881195:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573770.881199:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573770.881205:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcb28 00000400:00000010:0.0:1788573770.881212:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcb28. 00000100:00000001:0.0:1788573770.881217:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573770.881228:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f4a80 x1875440516613632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573770.881245:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573770.881248:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4a80 x1875440516613632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573792 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573770.881256:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573770.881259:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573770.881271:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573770.881274:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793d00. 00000400:00000200:0.0:1788573770.881297:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573770.881307:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573770.881314:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc7f8 00000400:00000010:0.0:1788573770.881318:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc7f8. 00000100:00000001:0.0:1788573770.881322:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573770.881331:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f4380 x1875440516613760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573770.881349:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573770.881395:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4380 x1875440516613760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573792 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573770.881410:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573770.881414:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573770.881425:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573770.881431:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793500. 00000400:00000200:0.0:1788573770.881437:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573770.881447:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573770.881452:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc990 00000400:00000010:0.0:1788573770.881461:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc990. 00000100:00000001:0.0:1788573770.881467:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573770.881476:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f6300 x1875440516613888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573770.881494:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573770.881502:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6300 x1875440516613888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573792 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573770.881514:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573770.881519:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573770.881530:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573770.881542:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793400. 00000400:00000200:0.0:1788573770.881547:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573770.881556:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573770.881563:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc088 00000400:00000010:0.0:1788573770.881575:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc088. 00000100:00000001:0.0:1788573770.881582:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573770.881590:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f7b80 x1875440516614016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573770.881610:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573770.881617:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7b80 x1875440516614016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573792 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573770.881632:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573770.881636:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788573770.881648:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788573770.881667:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793b00. 00000400:00000200:0.0:1788573770.881675:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788573770.881683:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788573770.881686:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc6e8 00000400:00000010:0.0:1788573770.881690:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc6e8. 00000100:00000001:0.0:1788573770.881694:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573770.881701:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f5880 x1875440516614144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573770.881716:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573770.881722:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5880 x1875440516614144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573792 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573770.881736:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573770.881739:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573771.392932:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573771.392937:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573771.392939:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573771.392943:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573771.392947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573771.392951:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573771.392954:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573771.392959:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573771.392966:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573771.648879:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309131501, at 4309131520 00000100:00000010:0.0:1788573771.648935:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea73c0. 00000020:00000001:0.0:1788573772.097024:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573772.097027:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573772.097030:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573772.416886:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573772.416891:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573772.416895:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573772.416902:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573772.416909:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573772.416914:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.416920:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573772.416925:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.416932:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.416963:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573772.416968:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573772.416972:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573772.416975:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573772.416978:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573772.416981:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.416983:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573772.416985:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.416988:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.929649:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573772.929659:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573772.929663:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573772.929676:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516614272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.929696:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573772.929700:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573772.929707:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573772.929712:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573772.929719:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573772.929725:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516614272:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573772.929732:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573772.929738:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573772.929749:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573772.929753:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573772.929760:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573772.929763:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573772.929772:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573772.929778:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573772.929786:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573772.929798:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516614272, portal 25 00000100:00000001:0.0:1788573772.929800:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573772.929806:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788573772.929857:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516614272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.929878:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573772.929888:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573772.929895:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573772.929900:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516614272, offset 0 00000400:00000200:0.0:1788573772.929915:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573772.929928:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573772.929937:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573772.929945:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573772.929953:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573772.929974:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573772.929984:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573772.929996:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546451:546451:256:4294967295] 192.168.202.128@tcp LPNI seq info [546374:546374:8:4294967295] 00000400:00000200:0.0:1788573772.930008:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573772.930026:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573772.930036:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573772.930045:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573772.930055:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573772.930060:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573772.930079:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573772.930089:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573772.930094:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573772.930100:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573772.930108:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573772.930111:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573772.930117:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573772.930123:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516614272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930133:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573772.930137:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516614272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930145:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.930147:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573772.930154:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573772.930159:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573772.930162:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309136622 00000100:00000010:0.0:1788573772.930167:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573772.930174:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573772.930180:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309136622 00000100:00000010:0.0:1788573772.930183:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50900. 00000100:00000001:0.0:1788573772.930186:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573772.930189:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573772.930193:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.930195:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573772.930200:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573772/real 1788573772] req@ffff8fc4901a4a80 x1875440516614272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573772.930210:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573772.930212:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573772.930214:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573772.930216:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573772.930217:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573772.930219:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573772.930227:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516614272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573772.930233:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516614272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930238:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573772.930239:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.930241:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573772.930242:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573772.930245:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516614272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930250:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573772.930254:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516614272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930259:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573772.930261:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.930262:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573772.930263:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.930265:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573772.930266:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573772.930268:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573772.930272:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573772.930274:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573772.930278:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516614272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573772.930286:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516614272:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573772.930291:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573772.930295:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516614400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930332:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573772.930334:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573772.930338:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573772.930341:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573772.930344:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573772.930348:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516614400:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573772.930351:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573772.930354:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573772.930356:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573772.930357:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573772.930361:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573772.930363:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573772.930368:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573772.930371:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573772.930375:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573772.930379:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516614400, portal 10 00000100:00000001:0.0:1788573772.930381:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573772.930383:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788573772.930390:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516614400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930398:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573772.930401:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573772.930403:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573772.930406:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516614400, offset 0 00000400:00000200:0.0:1788573772.930411:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573772.930416:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573772.930420:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573772.930422:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573772.930425:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573772.930426:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573772.930428:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573772.930435:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546452:546452:256:4294967295] 192.168.202.128@tcp LPNI seq info [546375:546375:8:4294967295] 00000400:00000200:0.0:1788573772.930438:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573772.930444:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573772.930448:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573772.930450:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573772.930453:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573772.930455:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573772.930463:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573772.930465:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573772.930467:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573772.930470:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573772.930473:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573772.930475:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573772.930478:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573772.930482:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516614400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930491:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573772.930494:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516614400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930501:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.930502:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573772.930506:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573772.930510:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573772.930514:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573772.930516:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309132819 00000100:00000001:0.0:1788573772.930519:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309136622 00000100:00000001:0.0:1788573772.930521:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309132819 00000100:00000001:0.0:1788573772.930523:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309136622 00000100:00000001:0.0:1788573772.930525:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573772.930532:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.930534:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573772.930539:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573772/real 1788573772] req@ffff8fc4901a7800 x1875440516614400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573772.930549:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573772.930550:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573772.930552:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573772.930555:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573772.930557:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573772.930560:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573772.930563:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516614400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573772.930571:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516614400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930576:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573772.930577:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.930578:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573772.930579:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573772.930582:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516614400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930586:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573772.930588:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516614400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930593:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573772.930595:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.930596:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573772.930597:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.930598:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573772.930600:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573772.930602:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573772.930610:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573772.930612:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573772.930617:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516614400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573772.930626:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516614400:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573772.930630:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573772.930635:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516614528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930641:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573772.930643:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573772.930646:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573772.930648:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573772.930650:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573772.930653:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516614528:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573772.930657:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573772.930665:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573772.930667:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573772.930669:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573772.930673:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573772.930675:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573772.930679:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573772.930682:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573772.930686:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573772.930689:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516614528, portal 10 00000100:00000001:0.0:1788573772.930691:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573772.930692:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788573772.930698:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516614528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930703:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573772.930709:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573772.930711:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573772.930713:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516614528, offset 0 00000400:00000200:0.0:1788573772.930716:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573772.930720:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573772.930724:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573772.930727:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573772.930730:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573772.930732:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573772.930734:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573772.930738:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546453:546453:256:4294967295] 192.168.202.128@tcp LPNI seq info [546376:546376:8:4294967295] 00000400:00000200:0.0:1788573772.930742:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573772.930750:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573772.930755:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573772.930759:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573772.930763:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573772.930766:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573772.930777:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573772.930778:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573772.930781:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573772.930783:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573772.930786:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573772.930787:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573772.930789:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573772.930792:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516614528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930798:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573772.930800:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516614528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930805:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.930830:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573772.930834:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573772.930836:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573772.930839:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573772.930840:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309132827 00000100:00000001:0.0:1788573772.930842:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309136622 00000100:00000001:0.0:1788573772.930844:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309132827 00000100:00000001:0.0:1788573772.930845:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309136622 00000100:00000001:0.0:1788573772.930847:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573772.930848:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.930850:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573772.930853:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573772/real 1788573772] req@ffff8fc4901a5180 x1875440516614528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573772.930858:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573772.930859:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573772.930860:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573772.930861:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573772.930863:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573772.930864:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573772.930866:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516614528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573772.930872:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516614528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930876:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573772.930878:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.930879:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573772.930880:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573772.930882:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516614528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930886:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573772.930892:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516614528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930897:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573772.930899:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.930900:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573772.930901:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.930902:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573772.930904:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573772.930906:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573772.930908:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573772.930909:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573772.930911:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516614528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573772.930916:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516614528:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573772.930919:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573772.930922:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516614656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930926:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573772.930927:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573772.930928:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573772.930930:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573772.930931:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573772.930933:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516614656:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573772.930935:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573772.930936:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573772.930937:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573772.930939:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573772.930941:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573772.930942:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573772.930951:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573772.930954:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573772.930958:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573772.930961:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516614656, portal 4 00000100:00000001:0.0:1788573772.930963:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573772.930964:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788573772.930971:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516614656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.930978:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573772.930980:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573772.930983:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573772.930985:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516614656, offset 0 00000400:00000200:0.0:1788573772.930989:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573772.930994:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573772.930998:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573772.931000:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573772.931003:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573772.931006:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573772.931008:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573772.931011:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546454:546454:256:4294967295] 192.168.202.128@tcp LPNI seq info [546377:546377:8:4294967295] 00000400:00000200:0.0:1788573772.931015:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573772.931021:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573772.931026:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573772.931029:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573772.931033:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573772.931036:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573772.931047:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573772.931050:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573772.931053:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573772.931056:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573772.931060:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573772.931067:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573772.931070:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573772.931073:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516614656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931079:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573772.931081:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516614656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931085:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.931086:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573772.931089:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573772.931092:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573772.931094:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573772.931096:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309132967 00000100:00000001:0.0:1788573772.931097:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309136622 00000100:00000001:0.0:1788573772.931099:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309132967 00000100:00000001:0.0:1788573772.931100:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309136622 00000100:00000001:0.0:1788573772.931101:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573772.931103:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.931104:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573772.931106:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573772/real 1788573772] req@ffff8fc4901a5c00 x1875440516614656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573772.931112:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573772.931113:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573772.931114:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573772.931116:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573772.931117:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573772.931118:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573772.931120:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516614656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573772.931126:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516614656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931133:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573772.931134:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931136:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573772.931137:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573772.931139:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516614656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931143:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573772.931146:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516614656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931151:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573772.931152:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931153:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573772.931154:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931155:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573772.931156:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573772.931158:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573772.931160:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573772.931161:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573772.931163:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516614656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573772.931168:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516614656:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573772.931170:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931172:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573772.931174:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516614272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931179:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573772.931181:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573772.931185:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573772.931187:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573772.931189:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573772.931190:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573772.931192:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573772.931194:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788573772.931197:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573772.931198:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573772.931199:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788573772.931202:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573772.931203:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931204:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573772.931207:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516614400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931211:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573772.931212:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573772.931213:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573772.931215:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573772.931215:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573772.931216:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573772.931217:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573772.931218:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7400. 02000000:00000001:0.0:1788573772.931219:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573772.931220:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573772.931221:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788573772.931223:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573772.931224:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931225:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573772.931227:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516614528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931230:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573772.931231:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573772.931232:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573772.931233:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573772.931234:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573772.931235:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573772.931238:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573772.931239:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788573772.931241:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573772.931242:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573772.931243:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788573772.931246:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573772.931247:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931248:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573772.931250:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516614656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931254:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573772.931255:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573772.931256:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573772.931257:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573772.931257:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573772.931258:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573772.931259:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573772.931260:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788573772.931261:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573772.931262:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573772.931262:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788573772.931264:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573772.931264:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931266:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931272:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573772.931274:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.931276:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573772.931278:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573772.931279:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573772.931283:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516614784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931289:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573772.931291:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573772.931293:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573772.931295:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573772.931297:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573772.931306:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516614784:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573772.931309:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573772.931326:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573772.931329:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573772.931331:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573772.931334:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573772.931336:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573772.931339:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573772.931341:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573772.931344:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573772.931350:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516614784, portal 4 00000100:00000001:0.0:1788573772.931352:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573772.931354:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788573772.931360:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516614784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931366:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573772.931369:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573772.931371:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573772.931374:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516614784, offset 0 00000400:00000200:0.0:1788573772.931378:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573772.931382:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573772.931386:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573772.931389:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573772.931392:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573772.931394:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573772.931397:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573772.931400:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546455:546455:256:4294967295] 192.168.202.128@tcp LPNI seq info [546378:546378:8:4294967295] 00000400:00000200:0.0:1788573772.931404:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573772.931410:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573772.931413:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573772.931416:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573772.931422:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573772.931425:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573772.931432:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573772.931433:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573772.931435:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573772.931438:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573772.931440:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573772.931441:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573772.931443:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573772.931446:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516614784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931452:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573772.931454:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516614784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931458:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.931459:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573772.931461:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573772.931464:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573772.931467:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573772.931469:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309132969 00000100:00000001:0.0:1788573772.931472:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309136622 00000100:00000001:0.0:1788573772.931474:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309132969 00000100:00000001:0.0:1788573772.931477:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309136622 00000100:00000001:0.0:1788573772.931480:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573772.931482:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.931484:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573772.931488:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573772/real 1788573772] req@ffff8fc4901a7100 x1875440516614784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573772.931498:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573772.931506:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573772.931508:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573772.931511:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573772.931513:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573772.931516:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573772.931519:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516614784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573772.931525:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516614784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931529:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573772.931530:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931531:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573772.931532:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573772.931534:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516614784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931539:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573772.931542:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516614784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931546:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573772.931547:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931548:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573772.931549:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931550:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573772.931552:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573772.931553:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573772.931555:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573772.931556:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573772.931559:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516614784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573772.931571:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516614784:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573772.931574:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931575:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573772.931577:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516614784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573797 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573772.931581:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573772.931582:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573772.931584:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573772.931585:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573772.931586:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573772.931587:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573772.931589:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573772.931590:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788573772.931591:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573772.931592:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573772.931593:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788573772.931595:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573772.931596:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931597:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573772.931600:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573772.931601:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573772.931602:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573772.931604:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573773.440966:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573773.440971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573773.440975:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573773.440979:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573773.440983:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573773.440987:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573773.440990:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573773.440993:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573773.440996:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573774.145010:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573774.145028:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573774.145030:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573774.464901:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573774.464906:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573774.464909:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573774.464913:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573774.464919:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573774.464926:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573774.464931:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573774.464935:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573774.464938:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573775.488908:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573775.488913:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573775.488916:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573775.488922:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573775.488928:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573775.488931:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573775.488941:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573775.488945:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573775.488948:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573776.512913:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573776.512922:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573776.512934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573776.512953:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573776.512962:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573776.512973:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573776.513012:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573776.513016:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573776.513019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573776.768902:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309136622, at 4309136640 00000100:00000010:0.0:1788573776.768916:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573777.217002:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573777.217005:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573777.217008:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573777.536992:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573777.537001:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573777.537009:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573777.537043:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573777.537048:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573777.537055:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573777.537059:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573777.537068:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573777.537073:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.049285:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573778.049297:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573778.049303:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573778.049321:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5c00 x1875440516614912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.049368:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573778.049371:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573778.049381:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573778.049387:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573778.049391:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573778.049403:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516614912:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573778.049417:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573778.049426:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573778.049432:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573778.049444:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573778.049457:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573778.049463:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573778.049477:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573778.049486:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573778.049498:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573778.049513:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516614912, portal 25 00000100:00000001:0.0:1788573778.049522:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573778.049532:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605327872 : -123400104223744 : ffff8fc4ac9f5c00) 00000100:00000040:0.0:1788573778.049546:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5c00 x1875440516614912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.049552:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573778.049562:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573778.049567:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573778.049589:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516614912, offset 0 00000400:00000200:0.0:1788573778.049602:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573778.049626:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573778.049642:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573778.049656:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573778.049669:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573778.049678:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573778.049683:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573778.049697:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546456:546456:256:4294967295] 192.168.202.128@tcp LPNI seq info [546379:546379:8:4294967295] 00000400:00000200:0.0:1788573778.049709:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573778.049726:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573778.049750:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573778.049762:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573778.049778:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573778.049791:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573778.049902:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573778.049907:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573778.049919:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573778.049935:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573778.049947:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573778.049950:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573778.049964:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573778.049976:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5c00 x1875440516614912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.049998:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573778.050011:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5c00 x1875440516614912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050043:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.050056:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573778.050066:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573778.050082:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573778.050101:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309141742 00000100:00000010:0.0:1788573778.050107:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573778.050117:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573778.050134:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309141742 00000100:00000010:0.0:1788573778.050147:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50120. 00000100:00000001:0.0:1788573778.050154:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573778.050159:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573778.050169:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.050178:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573778.050197:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573778/real 1788573778] req@ffff8fc4ac9f5c00 x1875440516614912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573778.050241:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573778.050244:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573778.050248:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573778.050257:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573778.050266:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573778.050274:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573778.050286:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5c00 x1875440516614912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573778.050326:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5c00 x1875440516614912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050387:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573778.050391:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.050395:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573778.050398:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573778.050403:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5c00 x1875440516614912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050410:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573778.050414:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5c00 x1875440516614912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050430:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573778.050432:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.050433:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573778.050434:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.050437:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573778.050438:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573778.050441:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573778.050444:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573778.050446:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573778.050449:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5c00 x1875440516614912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573778.050455:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516614912:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573778.050458:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.050459:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573778.050461:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5c00 x1875440516614912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050465:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573778.050467:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573778.050470:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573778.050472:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573778.050474:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573778.050476:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573778.050477:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573778.050479:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788573778.050482:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573778.050484:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573778.050485:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5c00. 00000100:00000001:0.0:1788573778.050487:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573778.050488:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.050490:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.050500:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573778.050501:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.050503:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573778.050505:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573778.050506:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573778.050509:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6d80 x1875440516615040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050515:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573778.050516:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573778.050518:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573778.050519:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573778.050521:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573778.050523:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516615040:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573778.050525:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573778.050527:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573778.050528:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573778.050529:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573778.050531:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573778.050533:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573778.050536:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573778.050538:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573778.050541:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573778.050544:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516615040, portal 10 00000100:00000001:0.0:1788573778.050545:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573778.050546:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605332352 : -123400104219264 : ffff8fc4ac9f6d80) 00000100:00000040:0.0:1788573778.050551:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6d80 x1875440516615040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050555:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573778.050557:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573778.050559:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573778.050560:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516615040, offset 0 00000400:00000200:0.0:1788573778.050563:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573778.050566:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573778.050573:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573778.050575:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573778.050577:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573778.050579:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573778.050581:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573778.050583:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546457:546457:256:4294967295] 192.168.202.128@tcp LPNI seq info [546380:546380:8:4294967295] 00000400:00000200:0.0:1788573778.050586:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573778.050591:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573778.050594:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573778.050596:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573778.050599:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573778.050601:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573778.050611:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573778.050612:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573778.050615:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573778.050619:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573778.050622:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573778.050624:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573778.050627:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573778.050631:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6d80 x1875440516615040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050640:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573778.050644:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6d80 x1875440516615040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050651:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.050653:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573778.050657:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573778.050661:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573778.050666:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573778.050668:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309137939 00000100:00000001:0.0:1788573778.050674:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309141742 00000100:00000001:0.0:1788573778.050676:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309137939 00000100:00000001:0.0:1788573778.050678:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309141742 00000100:00000001:0.0:1788573778.050679:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573778.050681:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.050682:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573778.050685:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573778/real 1788573778] req@ffff8fc4ac9f6d80 x1875440516615040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573778.050691:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573778.050692:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573778.050693:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573778.050695:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573778.050696:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573778.050698:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573778.050700:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6d80 x1875440516615040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573778.050706:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6d80 x1875440516615040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050710:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573778.050710:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.050712:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573778.050713:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573778.050715:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6d80 x1875440516615040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050719:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573778.050722:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6d80 x1875440516615040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050726:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573778.050728:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.050732:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573778.050733:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.050734:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573778.050736:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573778.050738:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573778.050740:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573778.050742:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573778.050744:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6d80 x1875440516615040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573778.050750:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516615040:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573778.050752:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573778.050755:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5500 x1875440516615168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050759:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573778.050760:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573778.050762:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573778.050763:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573778.050764:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573778.050766:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516615168:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573778.050768:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573778.050769:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573778.050771:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573778.050772:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573778.050774:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573778.050775:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573778.050778:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573778.050779:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573778.050781:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573778.050784:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516615168, portal 10 00000100:00000001:0.0:1788573778.050785:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573778.050791:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605326080 : -123400104225536 : ffff8fc4ac9f5500) 00000100:00000040:0.0:1788573778.050799:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5500 x1875440516615168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050805:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573778.050833:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573778.050837:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573778.050840:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516615168, offset 0 00000400:00000200:0.0:1788573778.050844:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573778.050847:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573778.050850:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573778.050852:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573778.050854:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573778.050855:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573778.050857:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573778.050859:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546458:546458:256:4294967295] 192.168.202.128@tcp LPNI seq info [546381:546381:8:4294967295] 00000400:00000200:0.0:1788573778.050862:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573778.050867:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573778.050870:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573778.050872:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573778.050875:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573778.050877:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573778.050884:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573778.050886:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573778.050888:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573778.050890:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573778.050893:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573778.050895:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573778.050898:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573778.050902:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5500 x1875440516615168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050921:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573778.050925:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5500 x1875440516615168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050932:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.050934:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573778.050938:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573778.050941:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573778.050945:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573778.050947:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309137947 00000100:00000001:0.0:1788573778.050949:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309141742 00000100:00000001:0.0:1788573778.050950:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309137947 00000100:00000001:0.0:1788573778.050952:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309141742 00000100:00000001:0.0:1788573778.050953:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573778.050955:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.050956:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573778.050959:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573778/real 1788573778] req@ffff8fc4ac9f5500 x1875440516615168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573778.050964:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573778.050966:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573778.050968:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573778.050970:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573778.050972:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573778.050974:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573778.050977:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5500 x1875440516615168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573778.050986:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5500 x1875440516615168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.050994:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573778.051001:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051003:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573778.051005:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573778.051009:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5500 x1875440516615168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051015:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573778.051018:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5500 x1875440516615168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051022:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573778.051024:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051025:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573778.051026:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051027:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573778.051029:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573778.051031:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573778.051033:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573778.051034:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573778.051036:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5500 x1875440516615168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573778.051042:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516615168:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573778.051045:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573778.051047:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4e00 x1875440516615296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051053:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573778.051055:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573778.051057:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573778.051059:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573778.051061:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573778.051071:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516615296:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573778.051075:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573778.051077:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573778.051079:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573778.051081:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573778.051084:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573778.051086:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573778.051090:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573778.051093:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573778.051097:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573778.051101:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516615296, portal 4 00000100:00000001:0.0:1788573778.051102:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573778.051103:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605324288 : -123400104227328 : ffff8fc4ac9f4e00) 00000100:00000040:0.0:1788573778.051107:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4e00 x1875440516615296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051112:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573778.051114:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573778.051115:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573778.051117:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516615296, offset 0 00000400:00000200:0.0:1788573778.051119:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573778.051122:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573778.051124:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573778.051126:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573778.051128:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573778.051130:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573778.051131:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573778.051134:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546459:546459:256:4294967295] 192.168.202.128@tcp LPNI seq info [546382:546382:8:4294967295] 00000400:00000200:0.0:1788573778.051136:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573778.051141:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573778.051144:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573778.051146:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573778.051153:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573778.051155:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573778.051162:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573778.051164:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573778.051166:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573778.051168:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573778.051171:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573778.051172:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573778.051174:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573778.051176:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4e00 x1875440516615296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051182:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573778.051184:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4e00 x1875440516615296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051188:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.051189:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573778.051191:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573778.051230:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573778.051233:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573778.051235:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309138087 00000100:00000001:0.0:1788573778.051237:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309141742 00000100:00000001:0.0:1788573778.051238:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309138087 00000100:00000001:0.0:1788573778.051240:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309141742 00000100:00000001:0.0:1788573778.051241:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573778.051243:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.051244:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573778.051247:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573778/real 1788573778] req@ffff8fc4ac9f4e00 x1875440516615296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573778.051253:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573778.051261:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573778.051264:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573778.051267:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573778.051268:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573778.051272:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573778.051277:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4e00 x1875440516615296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573778.051287:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4e00 x1875440516615296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051295:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573778.051297:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051299:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573778.051301:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573778.051305:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4e00 x1875440516615296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051312:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573778.051317:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4e00 x1875440516615296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051324:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573778.051326:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051328:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573778.051330:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051332:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573778.051334:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573778.051337:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573778.051341:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573778.051345:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573778.051350:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4e00 x1875440516615296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573778.051375:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516615296:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573778.051378:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573778.051380:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516615424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051385:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573778.051386:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573778.051388:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573778.051389:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573778.051391:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573778.051392:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516615424:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573778.051394:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573778.051396:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573778.051398:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573778.051399:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573778.051401:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573778.051402:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573778.051405:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573778.051407:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573778.051409:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573778.051411:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516615424, portal 4 00000100:00000001:0.0:1788573778.051413:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573778.051414:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788573778.051418:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516615424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051422:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573778.051424:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573778.051425:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573778.051426:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516615424, offset 0 00000400:00000200:0.0:1788573778.051429:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573778.051435:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573778.051438:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573778.051440:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573778.051442:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573778.051443:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573778.051445:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573778.051447:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546460:546460:256:4294967295] 192.168.202.128@tcp LPNI seq info [546383:546383:8:4294967295] 00000400:00000200:0.0:1788573778.051450:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573778.051454:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573778.051457:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573778.051459:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573778.051462:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573778.051464:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573778.051471:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573778.051472:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573778.051474:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573778.051476:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573778.051479:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573778.051480:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573778.051482:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573778.051484:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516615424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051489:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573778.051491:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516615424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573803 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051496:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.051497:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573778.051498:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573778.051501:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573778.051503:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573778.051508:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309138089 00000100:00000001:0.0:1788573778.051510:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309141742 00000100:00000001:0.0:1788573778.051511:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309138089 00000100:00000001:0.0:1788573778.051513:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309141742 00000100:00000001:0.0:1788573778.051515:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573778.051516:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.051517:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573778.051520:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573778/real 1788573778] req@ffff8fc4ac9f6a00 x1875440516615424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573778.051525:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573778.051526:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573778.051528:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573778.051529:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573778.051530:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573778.051531:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573778.051533:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516615424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573778.051539:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516615424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051543:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573778.051544:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051545:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573778.051546:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573778.051548:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516615424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051552:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573778.051554:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516615424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051558:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573778.051563:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051564:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573778.051565:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051566:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573778.051568:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573778.051570:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573778.051572:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573778.051573:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573778.051576:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516615424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573778.051581:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516615424:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573778.051583:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051585:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573778.051587:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6d80 x1875440516615040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051590:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573778.051592:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573778.051593:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788573778.051594:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573778.051595:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573778.051597:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573778.051598:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573778.051599:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35000. 02000000:00000001:0.0:1788573778.051601:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573778.051602:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573778.051603:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6d80. 00000100:00000001:0.0:1788573778.051605:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573778.051606:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051607:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573778.051609:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5500 x1875440516615168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051626:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573778.051627:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573778.051629:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573778.051631:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573778.051632:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573778.051634:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573778.051636:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573778.051637:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35800. 02000000:00000001:0.0:1788573778.051640:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573778.051641:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573778.051643:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5500. 00000100:00000001:0.0:1788573778.051645:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573778.051647:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051649:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573778.051653:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4e00 x1875440516615296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051660:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573778.051662:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573778.051663:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788573778.051666:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573778.051667:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573778.051668:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573778.051669:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573778.051670:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788573778.051671:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573778.051672:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573778.051673:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4e00. 00000100:00000001:0.0:1788573778.051675:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573778.051675:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051676:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573778.051679:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516615424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573803 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573778.051682:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573778.051683:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573778.051688:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788573778.051690:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573778.051691:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573778.051692:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573778.051693:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573778.051694:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788573778.051695:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573778.051696:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573778.051697:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788573778.051698:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573778.051699:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051700:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.051703:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573778.051703:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.051705:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573778.051706:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573778.561025:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573778.561032:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573778.561034:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573778.561039:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573778.561042:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573778.561047:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.561049:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573778.561051:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573778.561055:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573779.265017:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573779.265021:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573779.265024:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573779.584973:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573779.584981:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573779.584985:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573779.584993:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573779.584999:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573779.585003:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573779.585047:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573779.585051:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573779.585054:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573780.609038:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573780.609056:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573780.609060:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573780.609064:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573780.609068:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573780.609070:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573780.609072:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573780.609075:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573780.609078:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573781.888913:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309141742, at 4309141760 00000100:00000010:0.0:1788573781.888928:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573782.337059:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573782.337065:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573782.337070:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573782.656955:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573782.656971:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573782.656981:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573782.657007:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573782.657023:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573782.657035:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573782.657047:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573782.657058:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573782.657063:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573783.168953:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573783.168965:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573783.168968:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573783.168974:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573783.168979:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573783.168988:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14470 00000400:00000200:0.0:1788573783.168999:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573783.169007:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573783.169011:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573783.169015:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169019:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573783.169022:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169047:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573783.169053:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573783.169068:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573783.169072:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573783.169076:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573783.169079:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573783.169082:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573783.169087:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169089:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573783.169092:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169094:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573783.169098:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573783.169102:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169105:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573783.169108:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573783.169113:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573783.169130:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.169134:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573783.169136:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573783.169148:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573783.169152:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573783.169157:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573783.169159:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.169163:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573783.169167:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573783.169173:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573783.169185:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573783.169202:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573783.169215:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516615552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573783.169228:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573783.169234:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573783.169246:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516615552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573783.169278:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573783.169282:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573783.169286:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573783.169288:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573783.169290:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573783.169294:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573783.169299:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14470 00000400:00000200:0.0:1788573783.169305:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573783.169310:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573783.169313:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573783.169323:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169327:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573783.169334:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169338:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573783.169346:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573783.169350:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573783.169359:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573783.169365:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573783.169372:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573783.169384:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573783.169397:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169402:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573783.169404:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169406:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573783.169409:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573783.169413:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169421:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573783.169434:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573783.169445:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573783.169451:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.169459:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573783.169468:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573783.169477:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.169482:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573783.169491:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573783.169497:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573783.169503:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573783.169506:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.169516:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573783.169521:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573783.169531:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573783.169540:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573783.169550:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573783.169565:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516615680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573783.169592:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573783.169599:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573783.169609:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516615680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573783.169648:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573783.169659:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573783.169675:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573783.169678:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573783.169680:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573783.169684:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573783.169694:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14470 00000400:00000200:0.0:1788573783.169712:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573783.169722:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573783.169727:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573783.169754:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169766:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573783.169780:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169789:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573783.169798:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573783.169805:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573783.169884:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573783.169887:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573783.169892:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573783.169895:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573783.169908:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169933:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573783.169940:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169947:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573783.169966:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573783.169983:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573783.169996:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573783.170002:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573783.170012:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573783.170021:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.170034:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573783.170041:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573783.170052:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.170063:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573783.170077:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573783.170089:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573783.170099:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573783.170104:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.170112:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573783.170117:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573783.170135:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573783.170145:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573783.170160:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573783.170178:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516615808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573783.170244:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573783.170251:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573783.170272:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516615808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573783.170296:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573783.170304:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573783.170315:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573783.170319:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573783.170324:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573783.170330:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573783.170338:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14470 00000400:00000200:0.0:1788573783.170350:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573783.170361:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573783.170373:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573783.170392:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.170399:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573783.170411:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.170423:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573783.170432:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573783.170437:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573783.170445:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573783.170455:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573783.170471:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573783.170483:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573783.170499:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.170506:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573783.170522:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.170531:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573783.170546:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573783.170560:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573783.170571:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573783.170583:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573783.170590:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573783.170601:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.170610:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573783.170618:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573783.170632:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.170639:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573783.170652:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573783.170661:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573783.170668:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573783.170673:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.170683:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573783.170692:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573783.170699:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573783.170710:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573783.170719:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573783.170734:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516615936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573783.170773:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573783.170780:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573783.170795:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516615936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573783.170859:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573783.170871:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573783.170886:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573783.170895:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573783.170910:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573783.170924:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573783.170939:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14470 00000400:00000200:0.0:1788573783.170951:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573783.170965:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573783.170979:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573783.170993:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.171005:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573783.171015:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.171026:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573783.171039:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573783.171047:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573783.171054:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573783.171060:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573783.171068:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573783.171079:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573783.171096:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.171107:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573783.171116:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573783.171139:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573783.171146:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573783.171156:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573783.171164:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573783.171170:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573783.171178:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573783.171186:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.171193:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573783.171200:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573783.171211:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.171215:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573783.171236:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573783.171248:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573783.171254:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573783.171259:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573783.171263:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573783.171272:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573783.171281:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573783.171291:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573783.171301:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573783.171314:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516616064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573783.171333:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573783.171338:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573783.171349:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516616064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573783.171387:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573783.171394:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14480) 00000100:00000001:0.0:1788573783.680952:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573783.680962:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573783.680970:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573783.680978:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573783.680989:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573783.681001:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573783.681027:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573783.681033:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573783.681037:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573784.385027:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573784.385033:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573784.385037:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573784.704938:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573784.704951:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573784.704957:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573784.704973:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573784.704987:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573784.704994:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573784.705035:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573784.705044:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573784.705052:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573786.753074:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573786.753079:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573786.753083:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573786.753090:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573786.753096:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573786.753100:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573786.753106:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573786.753109:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573786.753112:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573787.008922:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309146862, at 4309146880 00000100:00000010:0.0:1788573787.008938:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea79c0. 00000020:00000001:0.0:1788573787.457003:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573787.457007:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573787.457010:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573787.776975:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573787.776981:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573787.776984:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573787.776991:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573787.777001:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573787.777005:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573787.777007:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573787.777009:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573787.777011:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573788.288904:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573788.288914:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573788.288917:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573788.288920:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573788.288924:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573788.288930:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14475 00000400:00000200:0.0:1788573788.288938:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573788.288945:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573788.288948:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573788.288968:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.288971:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573788.288974:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.288976:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573788.288981:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573788.288983:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573788.288986:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573788.288989:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573788.288992:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573788.288995:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573788.288999:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289002:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573788.289004:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289006:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573788.289010:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573788.289013:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289015:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573788.289017:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573788.289019:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573788.289022:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289023:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573788.289025:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573788.289046:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573788.289049:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573788.289052:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573788.289054:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289056:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573788.289058:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573788.289062:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573788.289069:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573788.289075:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573788.289083:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516616192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573788.289097:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573788.289100:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573788.289105:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516616192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573788.289122:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573788.289125:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573788.289127:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573788.289128:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573788.289130:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573788.289132:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573788.289133:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14475 00000400:00000200:0.0:1788573788.289143:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573788.289145:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573788.289147:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573788.289150:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289153:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573788.289156:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289158:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573788.289161:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573788.289162:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573788.289164:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573788.289166:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573788.289168:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573788.289170:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573788.289173:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289174:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573788.289176:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289178:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573788.289187:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573788.289190:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289192:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573788.289194:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573788.289200:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573788.289202:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289210:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573788.289215:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573788.289218:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289220:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573788.289224:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573788.289226:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573788.289229:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573788.289230:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289232:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573788.289233:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573788.289235:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573788.289239:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573788.289243:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573788.289248:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516616320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573788.289257:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573788.289259:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573788.289263:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516616320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573788.289271:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573788.289274:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573788.289277:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573788.289279:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573788.289280:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573788.289283:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573788.289291:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14475 00000400:00000200:0.0:1788573788.289296:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573788.289300:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573788.289302:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573788.289304:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289306:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573788.289307:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289309:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573788.289310:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573788.289311:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573788.289312:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573788.289313:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573788.289315:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573788.289316:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573788.289318:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289320:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573788.289321:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289322:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573788.289323:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573788.289325:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289326:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573788.289328:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573788.289330:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573788.289331:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289333:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573788.289334:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573788.289336:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289337:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573788.289339:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573788.289341:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573788.289342:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573788.289346:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289348:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573788.289349:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573788.289350:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573788.289353:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573788.289356:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573788.289359:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516616448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573788.289364:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573788.289366:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573788.289368:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516616448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573788.289373:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573788.289375:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573788.289378:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573788.289379:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573788.289380:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573788.289382:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573788.289383:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14475 00000400:00000200:0.0:1788573788.289386:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573788.289387:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573788.289389:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573788.289390:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289392:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573788.289393:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289395:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573788.289396:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573788.289397:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573788.289398:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573788.289399:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573788.289400:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573788.289405:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573788.289407:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289408:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573788.289410:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289411:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573788.289412:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573788.289414:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289415:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573788.289416:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573788.289418:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573788.289419:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289420:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573788.289422:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573788.289423:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289424:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573788.289426:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573788.289428:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573788.289429:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573788.289430:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289431:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573788.289432:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573788.289433:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573788.289436:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573788.289438:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573788.289441:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516616576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573788.289446:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573788.289447:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573788.289450:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516616576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573788.289454:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573788.289458:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573788.289461:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573788.289462:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573788.289463:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573788.289465:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573788.289467:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14475 00000400:00000200:0.0:1788573788.289469:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573788.289471:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573788.289472:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573788.289474:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289475:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573788.289476:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289477:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573788.289479:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573788.289479:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573788.289480:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573788.289482:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573788.289483:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573788.289484:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573788.289486:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289487:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573788.289488:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289490:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573788.289491:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573788.289492:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573788.289494:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573788.289495:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573788.289496:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573788.289497:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289498:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573788.289502:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573788.289504:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289505:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573788.289507:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573788.289508:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573788.289510:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573788.289510:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573788.289511:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573788.289512:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573788.289513:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573788.289516:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573788.289518:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573788.289520:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516616704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573788.289525:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573788.289526:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573788.289528:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516616704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573788.289533:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573788.289536:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14485) 00000100:00000001:0.0:1788573788.800936:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573788.800942:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573788.800954:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573788.800965:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573788.800971:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573788.800977:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573788.800987:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573788.800991:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573788.800995:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573789.504985:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573789.504988:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573789.504990:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573789.824931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573789.824937:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573789.824961:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573789.824966:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573789.824970:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573789.824973:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573789.824976:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573789.824979:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573789.824986:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573790.848936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573790.848941:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573790.848944:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573790.848957:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573790.848967:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573790.848973:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573790.849009:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573790.849012:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573790.849016:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573791.872898:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573791.872907:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573791.872909:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573791.872916:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573791.872921:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573791.872927:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573791.872934:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573791.872937:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573791.872943:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573792.128882:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309151982, at 4309152000 00000100:00000010:0.0:1788573792.128897:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f60. 00000020:00000001:0.0:1788573792.577015:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573792.577018:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573792.577021:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573792.896910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573792.896920:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573792.896925:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573792.896934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573792.896941:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573792.896946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573792.896964:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573792.896981:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573792.896984:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573793.920919:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573793.920926:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573793.920939:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573793.920948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573793.920953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573793.920958:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573793.920963:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573793.920969:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573793.920976:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573793.921006:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573793.921011:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573793.921015:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573793.921021:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573793.921027:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573793.921031:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573793.921037:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573793.921041:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573793.921044:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573794.625019:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573794.625024:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573794.625027:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573794.945019:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573794.945028:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573794.945033:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573794.945036:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573794.945038:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573794.945043:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573794.945045:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573794.945048:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573794.945050:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573795.968887:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573795.968909:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573795.968914:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573795.968924:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573795.968932:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573795.968944:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573795.968967:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573795.968970:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573795.968973:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573796.992921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573796.992930:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573796.992935:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573796.992942:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573796.992952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573796.992956:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573796.992961:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573796.992965:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573796.992970:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573797.248926:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309157102, at 4309157120 00000100:00000010:0.0:1788573797.248942:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea77e0. 00000020:00000001:0.0:1788573797.697031:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573797.697035:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573797.697038:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573798.016919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573798.016937:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573798.016942:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573798.016948:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573798.016956:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573798.016961:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573798.016969:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573798.016973:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573798.016978:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573798.017022:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573798.017027:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573798.017030:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573798.017037:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573798.017041:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573798.017045:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573798.017049:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573798.017051:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573798.017054:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573799.040930:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573799.040937:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573799.040958:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573799.040963:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573799.040968:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573799.040974:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573799.040989:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573799.040991:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573799.040993:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573799.041034:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573799.041041:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573799.041045:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573799.041049:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573799.041053:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573799.041055:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573799.041060:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573799.041063:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573799.041065:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573799.745046:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573799.745049:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573799.745052:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573800.064924:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573800.064931:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573800.064936:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573800.064941:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573800.064949:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573800.064957:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573800.064964:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573800.064968:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573800.064972:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573801.088899:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573801.088907:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573801.088911:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573801.088917:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573801.088925:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573801.088931:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573801.088935:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573801.088939:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573801.088943:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573801.088990:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573801.089000:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573801.089002:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573801.089006:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573801.089011:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573801.089015:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573801.089020:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573801.089023:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573801.089025:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573802.112929:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573802.112935:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573802.112939:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573802.112944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573802.112950:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573802.112956:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573802.112967:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573802.112972:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573802.112979:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573802.368907:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309162222, at 4309162240 00000100:00000010:0.0:1788573802.368923:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7540. 00000020:00000001:0.0:1788573802.816991:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573802.816994:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573802.816997:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573803.136949:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573803.136960:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573803.136964:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573803.136981:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573803.136993:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573803.137002:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573803.137020:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573803.137026:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573803.137034:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788573803.649061:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573803.649078:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573803.649083:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573803.649087:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573804.160909:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573804.160918:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573804.160936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573804.160940:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573804.160946:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573804.160950:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573804.160958:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573804.160962:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573804.160966:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573804.865001:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573804.865004:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573804.865007:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573805.184928:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573805.184933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573805.184936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573805.184942:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573805.184949:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573805.184953:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573805.184984:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573805.184989:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573805.184994:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573806.208966:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573806.208974:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573806.208978:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573806.208989:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573806.209002:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573806.209011:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573806.209018:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573806.209022:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573806.209027:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573807.232930:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573807.232936:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573807.232940:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573807.232946:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573807.232954:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573807.232960:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573807.232977:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573807.232981:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573807.232985:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573807.488897:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309167342, at 4309167360 00000100:00000010:0.0:1788573807.488922:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7a80. 00000020:00000001:0.0:1788573807.937034:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573807.937041:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573807.937046:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573808.257012:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573808.257019:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573808.257030:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573808.257046:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573808.257061:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573808.257071:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573808.257094:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573808.257104:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573808.257110:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573809.280954:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573809.280959:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573809.280962:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573809.280970:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573809.280975:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573809.280978:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573809.280980:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573809.280982:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573809.280984:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573809.984997:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573809.985000:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573809.985002:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573810.304930:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573810.304938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573810.304941:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573810.304950:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573810.304954:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573810.304959:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573810.304963:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573810.304967:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573810.304970:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573810.305144:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573810.305149:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573810.305151:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573810.305163:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573810.305166:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573810.305168:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573810.305170:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573810.305172:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573810.305174:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573811.329030:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573811.329034:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573811.329035:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573811.329038:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573811.329040:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573811.329042:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573811.329044:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573811.329045:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573811.329047:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573812.352938:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573812.352946:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573812.352951:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573812.352960:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573812.352968:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573812.352974:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573812.352988:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573812.352991:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573812.352993:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573812.608921:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309172461, at 4309172480 00000100:00000010:0.0:1788573812.608940:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c60. 00000020:00000001:0.0:1788573813.057032:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573813.057038:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573813.057042:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573813.376943:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573813.376954:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573813.376958:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573813.376967:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573813.376976:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573813.376984:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573813.376992:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573813.376998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573813.377012:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573814.401001:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573814.401010:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573814.401020:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573814.401030:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573814.401040:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573814.401046:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573814.401082:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573814.401088:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573814.401093:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573814.401127:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573814.401132:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573814.401134:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573814.401137:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573814.401139:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573814.401141:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573814.401144:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573814.401147:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573814.401149:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573815.105016:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573815.105019:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573815.105022:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573815.424953:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573815.424964:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573815.424968:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573815.424977:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573815.424985:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573815.424991:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573815.424996:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573815.425001:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573815.425005:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573816.448927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573816.448935:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573816.448938:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573816.448950:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573816.448957:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573816.448960:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573816.448970:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573816.448979:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573816.448982:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573817.472932:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573817.472936:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573817.472941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573817.472946:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573817.472950:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573817.472956:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573817.472972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573817.472975:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573817.472978:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573817.728881:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309177581, at 4309177600 00000100:00000010:0.0:1788573817.728897:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7420. 00000020:00000001:0.0:1788573818.176982:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573818.176987:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573818.176990:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573818.496913:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573818.496919:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573818.496923:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573818.496933:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573818.496941:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573818.496947:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573818.496972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573818.496975:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573818.496979:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573819.520911:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573819.520920:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573819.520926:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573819.520936:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573819.520941:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573819.520955:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573819.520969:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573819.520973:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573819.520977:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573820.224952:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573820.224956:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573820.224959:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573820.544901:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573820.544910:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573820.544914:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573820.544923:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573820.544930:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573820.544938:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573820.544951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573820.544954:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573820.544960:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573821.568907:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573821.568917:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573821.568922:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573821.568930:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573821.568939:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573821.568946:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573821.568960:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573821.568965:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573821.568970:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573822.592919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573822.592926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573822.592930:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573822.592938:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573822.592946:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573822.592951:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573822.592964:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573822.592968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573822.592975:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573822.848902:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309182701, at 4309182720 00000100:00000010:0.0:1788573822.848919:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ba0. 00000020:00000001:0.0:1788573823.297039:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573823.297042:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573823.297045:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573823.616908:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573823.616912:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573823.616915:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573823.616921:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573823.616928:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573823.616948:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573823.616969:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573823.616981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573823.616985:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573824.640935:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573824.640944:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573824.640947:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573824.640954:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573824.640961:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573824.640966:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573824.640972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573824.640984:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573824.640994:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573824.641041:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573824.641046:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573824.641048:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573824.641051:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573824.641055:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573824.641057:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573824.641062:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573824.641064:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573824.641068:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573825.345031:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573825.345036:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573825.345040:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573825.664894:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573825.664900:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573825.664904:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573825.664910:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573825.664917:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573825.664924:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573825.664931:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573825.664936:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573825.664941:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573825.664976:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573825.664980:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573825.664982:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573825.664991:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573825.665004:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573825.665006:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573825.665009:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573825.665010:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573825.665013:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573826.688936:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573826.688942:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573826.688946:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573826.688953:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573826.688960:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573826.688965:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573826.688977:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573826.688982:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573826.688987:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573826.689011:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573826.689016:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573826.689018:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573826.689021:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573826.689026:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573826.689029:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573826.689032:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573826.689034:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573826.689037:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573827.968888:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309187821, at 4309187840 00000100:00000010:0.0:1788573827.968904:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7600. 00000020:00000001:0.0:1788573828.416971:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573828.416976:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573828.416979:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788573829.248950:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573829.248964:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573829.248966:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573829.248971:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573829.248976:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573829.248982:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14516 00000400:00000200:0.0:1788573829.248990:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573829.249013:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573829.249015:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573829.249018:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249020:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573829.249022:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249024:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573829.249027:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573829.249028:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573829.249030:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573829.249033:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573829.249035:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573829.249036:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573829.249038:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249040:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573829.249041:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249042:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573829.249044:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573829.249046:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249048:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573829.249051:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573829.249053:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573829.249055:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249057:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573829.249058:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573829.249063:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573829.249066:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573829.249069:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573829.249070:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249072:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573829.249074:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573829.249079:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573829.249084:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573829.249094:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573829.249102:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516621312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573829.249110:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573829.249113:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573829.249117:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516621312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573829.249135:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573829.249137:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573829.249140:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573829.249141:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573829.249142:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573829.249144:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573829.249146:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14516 00000400:00000200:0.0:1788573829.249150:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573829.249153:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573829.249155:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573829.249158:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249160:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573829.249164:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249166:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573829.249169:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573829.249171:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573829.249175:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573829.249177:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573829.249179:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573829.249181:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573829.249185:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249190:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573829.249197:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249200:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573829.249204:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573829.249210:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249214:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573829.249219:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573829.249226:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573829.249232:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249235:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573829.249239:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573829.249246:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249249:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573829.249253:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573829.249267:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573829.249269:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573829.249270:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249272:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573829.249273:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573829.249274:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573829.249278:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573829.249283:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573829.249288:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516621440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573829.249296:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573829.249300:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573829.249303:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516621440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573829.249311:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573829.249313:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573829.249330:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573829.249333:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573829.249342:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573829.249349:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573829.249354:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14516 00000400:00000200:0.0:1788573829.249361:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573829.249366:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573829.249369:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573829.249372:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249375:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573829.249378:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249382:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573829.249391:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573829.249394:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573829.249395:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573829.249397:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573829.249398:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573829.249401:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573829.249407:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249410:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573829.249414:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249418:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573829.249420:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573829.249426:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249430:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573829.249433:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573829.249438:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573829.249444:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249448:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573829.249451:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573829.249456:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249459:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573829.249469:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573829.249474:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573829.249478:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573829.249479:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249484:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573829.249485:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573829.249487:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573829.249493:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573829.249499:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573829.249509:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516621568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573829.249523:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573829.249527:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573829.249535:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516621568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573829.249545:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573829.249547:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573829.249549:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573829.249551:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573829.249552:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573829.249555:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573829.249560:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14516 00000400:00000200:0.0:1788573829.249565:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573829.249569:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573829.249573:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573829.249579:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249584:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573829.249588:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249592:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573829.249593:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573829.249601:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573829.249604:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573829.249606:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573829.249610:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573829.249614:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573829.249619:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249622:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573829.249628:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249633:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573829.249640:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573829.249648:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249655:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573829.249658:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573829.249664:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573829.249667:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249669:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573829.249674:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573829.249681:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249686:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573829.249692:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573829.249696:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573829.249699:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573829.249702:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249705:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573829.249708:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573829.249712:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573829.249718:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573829.249725:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573829.249733:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516621696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573829.249751:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573829.249754:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573829.249766:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516621696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573829.249786:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573829.249789:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573829.249798:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573829.249802:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573829.249805:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573829.249836:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573829.249842:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14516 00000400:00000200:0.0:1788573829.249852:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573829.249857:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573829.249860:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573829.249866:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249871:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573829.249875:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249879:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573829.249884:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573829.249888:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573829.249891:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573829.249894:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573829.249898:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573829.249901:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573829.249906:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249910:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573829.249914:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249917:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573829.249921:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573829.249927:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573829.249934:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573829.249946:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573829.249952:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573829.249958:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249964:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573829.249968:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573829.249975:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.249979:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573829.249984:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573829.249989:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573829.249993:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573829.249996:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573829.250002:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573829.250007:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573829.250012:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573829.250020:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573829.250027:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573829.250039:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516621824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573829.250071:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573829.250077:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573829.250090:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516621824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573829.250112:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573829.250121:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14526) 00000100:00000001:0.0:1788573829.760958:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573829.760962:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573829.760966:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573829.760972:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573829.760977:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573829.760980:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573829.760989:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573829.760992:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573829.760995:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573830.465028:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573830.465051:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573830.465056:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573830.784962:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573830.784977:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573830.784982:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573830.784998:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573830.785008:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573830.785014:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573830.785017:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573830.785019:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573830.785023:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573831.808956:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573831.808961:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573831.808965:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573831.808971:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573831.808979:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573831.808989:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573831.808996:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573831.809230:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573831.809234:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573831.809266:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573831.809282:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573831.809285:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573831.809287:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573831.809289:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573831.809292:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573831.809295:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573831.809297:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573831.809300:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573832.832982:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573832.832987:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573832.832995:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573832.833012:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573832.833019:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573832.833027:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573832.833037:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573832.833043:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573832.833053:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573833.088900:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309192941, at 4309192960 00000100:00000010:0.0:1788573833.088927:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ea0. 00000020:00000001:0.0:1788573833.536992:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573833.536996:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573833.536998:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573833.856991:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573833.856996:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573833.856998:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573833.857002:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573833.857005:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573833.857008:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573833.857018:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573833.857027:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573833.857045:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573834.880935:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573834.880957:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573834.880960:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573834.880965:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573834.880969:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573834.880972:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573834.880983:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573834.880992:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573834.881009:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573835.584962:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573835.584965:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573835.584968:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573836.928933:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573836.928940:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573836.928943:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573836.928951:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573836.928966:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573836.928973:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573836.928980:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573836.928988:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573836.928993:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573837.952929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573837.952938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573837.952959:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573837.952965:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573837.952971:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573837.952977:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573837.952997:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573837.953000:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573837.953012:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573838.208925:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309198061, at 4309198080 00000100:00000010:0.0:1788573838.208942:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c00. 00000020:00000001:0.0:1788573838.657029:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573838.657033:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573838.657036:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573838.976954:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573838.976961:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573838.976965:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573838.976972:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573838.976980:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573838.976990:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573838.977006:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573838.977012:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573838.977024:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573840.000937:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573840.000946:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573840.000952:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573840.000966:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573840.000978:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573840.000987:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573840.000992:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573840.000997:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573840.001002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573840.705069:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573840.705073:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573840.705077:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573841.024956:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573841.024961:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573841.024965:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573841.024971:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573841.024984:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573841.024989:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573841.024994:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573841.024997:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573841.025002:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788573841.537210:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573841.537233:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788573841.537244:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788573841.537248:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573842.048932:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573842.048940:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573842.048948:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573842.048963:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573842.048976:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573842.048986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573842.049007:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573842.049013:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573842.049018:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573843.072947:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573843.072951:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573843.072953:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573843.072956:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573843.072962:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573843.072967:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573843.072973:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573843.072976:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573843.072980:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573843.328911:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309203183, at 4309203200 00000100:00000010:0.0:1788573843.328927:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f00. 00000020:00000001:0.0:1788573843.776970:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573843.776973:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573843.776976:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573844.096904:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573844.096914:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573844.096919:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573844.096929:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573844.096937:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573844.096961:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573844.096975:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573844.096978:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573844.096982:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573844.608927:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573844.608941:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573844.608953:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573844.608958:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573844.608964:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573844.608972:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14532 00000400:00000200:0.0:1788573844.608982:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573844.608989:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573844.608992:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573844.608996:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.608999:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573844.609002:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609004:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573844.609009:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573844.609011:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573844.609014:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573844.609017:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573844.609020:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573844.609023:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573844.609026:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609028:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573844.609030:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609033:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573844.609035:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573844.609039:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609042:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573844.609045:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573844.609080:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573844.609083:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.609086:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573844.609088:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573844.609095:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573844.609099:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573844.609104:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573844.609105:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.609109:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573844.609111:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573844.609116:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573844.609123:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573844.609130:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573844.609141:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516623232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573844.609153:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573844.609157:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573844.609163:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516623232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573844.609191:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573844.609195:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573844.609199:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573844.609201:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573844.609203:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573844.609206:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573844.609209:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14532 00000400:00000200:0.0:1788573844.609216:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573844.609224:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573844.609230:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573844.609233:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609236:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573844.609248:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609253:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573844.609258:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573844.609262:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573844.609265:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573844.609269:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573844.609273:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573844.609279:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573844.609287:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609292:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573844.609298:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609304:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573844.609310:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573844.609317:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609321:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573844.609326:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573844.609332:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573844.609336:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.609339:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573844.609346:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573844.609350:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.609354:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573844.609360:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573844.609364:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573844.609371:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573844.609373:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.609375:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573844.609377:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573844.609381:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573844.609388:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573844.609396:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573844.609403:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516623360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573844.609445:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573844.609449:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573844.609457:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516623360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573844.609470:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573844.609476:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573844.609479:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573844.609481:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573844.609484:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573844.609487:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573844.609492:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14532 00000400:00000200:0.0:1788573844.609500:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573844.609506:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573844.609508:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573844.609510:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609513:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573844.609516:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609521:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573844.609523:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573844.609526:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573844.609530:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573844.609533:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573844.609537:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573844.609541:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573844.609546:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609548:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573844.609552:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609554:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573844.609560:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573844.609564:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609569:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573844.609570:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573844.609573:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573844.609576:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.609588:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573844.609591:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573844.609594:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.609596:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573844.609600:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573844.609604:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573844.609608:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573844.609610:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.609613:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573844.609614:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573844.609616:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573844.609621:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573844.609628:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573844.609633:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516623488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573844.609641:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573844.609645:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573844.609650:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516623488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573844.609658:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573844.609662:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573844.609668:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573844.609671:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573844.609674:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573844.609677:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573844.609687:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14532 00000400:00000200:0.0:1788573844.609695:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573844.609699:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573844.609702:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573844.609707:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609711:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573844.609714:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609716:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573844.609718:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573844.609719:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573844.609720:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573844.609721:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573844.609722:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573844.609726:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573844.609730:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609731:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573844.609733:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609738:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573844.609740:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573844.609746:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609748:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573844.609750:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573844.609754:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573844.609756:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.609759:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573844.609761:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573844.609766:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.609770:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573844.609781:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573844.609785:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573844.609789:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573844.609794:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.609798:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573844.609800:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573844.609803:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573844.609843:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573844.609851:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573844.609863:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516623616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573844.609872:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573844.609877:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573844.609884:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516623616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573844.609902:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573844.609905:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573844.609913:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573844.609918:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573844.609923:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573844.609927:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573844.609933:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14532 00000400:00000200:0.0:1788573844.609945:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573844.609953:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573844.609959:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573844.609965:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609969:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573844.609975:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.609980:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573844.609984:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573844.609986:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573844.609989:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573844.609994:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573844.610000:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573844.610013:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573844.610023:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.610029:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573844.610035:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573844.610042:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573844.610060:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573844.610069:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573844.610076:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573844.610082:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573844.610087:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573844.610092:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.610095:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573844.610098:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573844.610103:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.610107:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573844.610113:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573844.610116:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573844.610122:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573844.610125:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573844.610130:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573844.610132:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573844.610136:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573844.610144:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573844.610150:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573844.610160:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516623744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573844.610169:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573844.610171:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573844.610175:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516623744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573844.610188:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573844.610200:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14542) 00000100:00000001:0.0:1788573845.120934:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573845.120939:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573845.120942:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573845.120946:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573845.120950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573845.120957:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573845.120973:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573845.120977:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573845.120983:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573845.824933:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573845.824936:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573845.824939:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573846.144911:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573846.144919:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573846.144922:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573846.144929:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573846.144935:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573846.144939:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573846.144951:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573846.144957:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573846.144961:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573847.168913:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573847.168920:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573847.168925:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573847.168934:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573847.168942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573847.168947:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573847.168960:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573847.168964:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573847.168972:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573848.192911:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573848.192920:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573848.192924:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573848.192931:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573848.192936:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573848.192943:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573848.192947:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573848.192957:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573848.192962:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573848.448957:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309208302, at 4309208320 00000100:00000010:0.0:1788573848.449009:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7840. 00000020:00000001:0.0:1788573848.897071:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573848.897077:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573848.897082:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573849.216915:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573849.216924:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573849.216927:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573849.216932:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573849.216937:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573849.216944:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573849.216980:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573849.216992:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573849.217006:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.217064:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573849.217069:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573849.217072:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573849.217076:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573849.217079:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573849.217082:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573849.217085:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573849.217086:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573849.217087:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573849.728925:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573849.728936:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573849.728940:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573849.728955:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573849.728962:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573849.728971:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14537 00000400:00000200:0.0:1788573849.728981:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573849.728996:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573849.729000:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573849.729004:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729029:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573849.729032:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729035:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573849.729039:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573849.729042:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573849.729045:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573849.729049:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573849.729052:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573849.729055:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573849.729058:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729060:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573849.729063:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729065:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573849.729068:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573849.729071:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729074:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573849.729077:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573849.729081:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573849.729083:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729085:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573849.729087:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573849.729092:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573849.729095:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573849.729098:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573849.729100:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729104:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573849.729106:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573849.729111:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573849.729119:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573849.729125:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573849.729136:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516623872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573849.729160:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573849.729164:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573849.729170:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516623872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573849.729192:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573849.729196:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573849.729199:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573849.729201:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573849.729203:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573849.729206:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573849.729209:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14537 00000400:00000200:0.0:1788573849.729214:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573849.729218:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573849.729220:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573849.729222:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729224:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573849.729226:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729228:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573849.729232:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573849.729234:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573849.729238:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573849.729240:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573849.729244:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573849.729246:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573849.729251:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729253:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573849.729257:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729261:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573849.729270:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573849.729276:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729281:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573849.729283:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573849.729289:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573849.729293:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729296:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573849.729301:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573849.729305:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729307:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573849.729311:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573849.729315:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573849.729317:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573849.729320:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729322:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573849.729323:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573849.729325:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573849.729330:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573849.729336:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573849.729342:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516624000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573849.729350:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573849.729353:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573849.729358:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516624000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573849.729368:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573849.729371:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573849.729374:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573849.729378:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573849.729380:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573849.729382:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573849.729389:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14537 00000400:00000200:0.0:1788573849.729394:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573849.729399:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573849.729401:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573849.729406:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729410:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573849.729412:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729414:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573849.729417:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573849.729420:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573849.729422:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573849.729424:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573849.729426:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573849.729431:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573849.729438:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729441:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573849.729443:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729445:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573849.729451:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573849.729457:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729461:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573849.729467:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573849.729473:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573849.729478:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729483:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573849.729486:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573849.729489:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729491:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573849.729496:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573849.729500:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573849.729502:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573849.729509:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729512:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573849.729513:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573849.729517:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573849.729523:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573849.729528:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573849.729535:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516624128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573849.729546:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573849.729550:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573849.729558:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516624128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573849.729571:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573849.729575:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573849.729579:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573849.729581:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573849.729583:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573849.729585:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573849.729589:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14537 00000400:00000200:0.0:1788573849.729604:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573849.729610:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573849.729612:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573849.729614:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729616:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573849.729621:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729623:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573849.729624:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573849.729625:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573849.729628:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573849.729631:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573849.729636:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573849.729643:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573849.729649:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729653:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573849.729656:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729659:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573849.729663:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573849.729665:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729667:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573849.729668:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573849.729670:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573849.729671:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729673:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573849.729675:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573849.729677:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729680:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573849.729683:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573849.729686:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573849.729688:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573849.729690:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729694:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573849.729695:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573849.729697:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573849.729700:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573849.729705:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573849.729712:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516624256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573849.729720:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573849.729723:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573849.729729:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516624256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573849.729738:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573849.729746:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573849.729753:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573849.729755:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573849.729757:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573849.729762:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573849.729766:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14537 00000400:00000200:0.0:1788573849.729770:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573849.729775:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573849.729779:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573849.729784:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729789:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573849.729790:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729792:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573849.729794:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573849.729794:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573849.729795:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573849.729797:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573849.729799:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573849.729803:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573849.729828:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729830:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573849.729831:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729833:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573849.729835:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573849.729837:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573849.729838:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573849.729842:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573849.729845:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573849.729849:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729851:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573849.729860:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573849.729864:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729868:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573849.729875:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573849.729881:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573849.729887:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573849.729890:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573849.729893:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573849.729896:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573849.729899:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573849.729905:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573849.729911:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573849.729921:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516624384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573849.729931:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573849.729935:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573849.729940:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516624384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573849.729963:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573849.729969:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14547) 00000100:00000001:0.0:1788573850.240988:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573850.240993:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573850.240995:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573850.241000:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573850.241017:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573850.241024:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573850.241071:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573850.241074:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573850.241080:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573850.944990:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573850.944994:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573850.944997:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573851.264952:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573851.264961:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573851.264980:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573851.264986:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573851.264991:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573851.264996:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573851.265010:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573851.265013:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573851.265017:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573852.288956:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573852.288959:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573852.288961:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573852.288964:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573852.288967:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573852.288970:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573852.288972:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573852.288974:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573852.288977:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573853.312889:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573853.312898:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573853.312901:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573853.312908:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573853.312916:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573853.312922:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573853.312927:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573853.312930:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573853.312933:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573853.312963:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573853.312968:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573853.312972:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573853.312977:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573853.312984:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573853.312989:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573853.313003:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573853.313007:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573853.313010:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573853.568951:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309213422, at 4309213440 00000100:00000010:0.0:1788573853.568971:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7900. 00000020:00000001:0.0:1788573854.017014:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573854.017035:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573854.017038:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573854.336918:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573854.336924:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573854.336927:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573854.336937:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573854.336943:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573854.336947:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573854.336962:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573854.336965:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573854.336969:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573855.360927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573855.360933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573855.360937:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573855.360946:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573855.360952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573855.360959:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573855.360975:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573855.360980:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573855.360983:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573856.064997:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573856.065000:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573856.065003:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573856.384935:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573856.384940:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573856.384943:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573856.384951:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573856.384957:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573856.384962:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573856.384968:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573856.384973:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573856.384977:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573856.384997:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573856.385001:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573856.385004:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573856.385007:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573856.385011:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573856.385014:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573856.385024:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573856.385025:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573856.385028:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573857.408930:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573857.408935:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573857.408937:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573857.408942:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573857.408948:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573857.408953:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573857.408969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573857.408974:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573857.408977:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573857.408995:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573857.408999:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573857.409001:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573857.409003:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573857.409006:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573857.409008:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573857.409011:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573857.409012:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573857.409015:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573858.432909:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573858.432924:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573858.432927:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573858.432933:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573858.432939:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573858.432944:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573858.432967:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573858.432970:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573858.432974:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573858.688895:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309218541, at 4309218560 00000100:00000010:0.0:1788573858.688912:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7720. 00000020:00000001:0.0:1788573859.137028:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573859.137031:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573859.137033:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573859.456911:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573859.456921:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573859.456943:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573859.456950:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573859.456956:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573859.456962:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573859.456977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573859.456981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573859.456985:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573860.480929:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573860.480935:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573860.480940:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573860.480947:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573860.480955:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573860.480961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573860.480967:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573860.480971:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573860.480975:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573861.184980:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573861.184984:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573861.184987:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573861.504925:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573861.504935:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573861.504938:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573861.504948:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573861.504955:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573861.504961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573861.504969:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573861.504972:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573861.504974:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573862.529021:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573862.529034:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573862.529047:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573862.529053:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573862.529070:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573862.529077:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573862.529097:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573862.529101:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573862.529105:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573863.552928:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573863.552948:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573863.552954:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573863.552963:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573863.552970:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573863.552977:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573863.552983:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573863.552989:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573863.552994:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573863.808914:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309223661, at 4309223680 00000100:00000010:0.0:1788573863.808931:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea78a0. 00000020:00000001:0.0:1788573864.256947:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573864.256950:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573864.256953:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573864.576921:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573864.576927:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573864.576932:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573864.576940:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573864.576947:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573864.576954:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573864.576959:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573864.576964:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573864.576970:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573865.600927:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573865.600932:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573865.600935:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573865.600942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573865.600946:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573865.600951:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573865.600966:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573865.600969:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573865.600973:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573866.305041:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573866.305048:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573866.305051:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573866.624962:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573866.624967:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573866.624971:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573866.624997:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573866.625004:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573866.625014:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573866.625027:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573866.625031:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573866.625037:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573867.648935:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573867.648943:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573867.648947:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573867.648955:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573867.648962:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573867.648970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573867.648982:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573867.648990:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573867.648998:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573867.649049:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573867.649052:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573867.649054:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573867.649056:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573867.649059:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573867.649060:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573867.649062:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573867.649065:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573867.649068:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573868.672959:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573868.672971:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573868.672975:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573868.672984:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573868.672991:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573868.672996:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573868.673002:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573868.673008:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573868.673014:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573868.929028:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309228781, at 4309228800 00000100:00000010:0.0:1788573868.929041:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7720. 00000020:00000001:0.0:1788573869.376985:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573869.376990:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573869.377022:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573869.696891:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573869.696900:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573869.696903:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573869.696909:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573869.696915:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573869.696920:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573869.696927:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573869.696932:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573869.696937:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573869.696976:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573869.696979:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573869.696982:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573869.696985:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573869.696989:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573869.696993:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573869.696996:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573869.696999:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573869.697001:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573870.720942:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573870.720953:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573870.720957:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573870.720965:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573870.720974:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573870.720982:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573870.721003:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573870.721007:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573870.721011:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573871.425114:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573871.425118:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573871.425122:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573872.768973:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573872.768985:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573872.768989:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573872.768999:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573872.769007:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573872.769020:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573872.769025:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573872.769045:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573872.769054:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573873.792906:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573873.792915:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573873.792919:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573873.792925:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573873.792932:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573873.792937:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573873.792957:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573873.792960:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573873.792962:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573874.048893:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309233902, at 4309233920 00000100:00000010:0.0:1788573874.048910:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7900. 00000020:00000001:0.0:1788573874.496970:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573874.496974:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573874.496978:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573874.816959:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573874.816966:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573874.816972:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573874.816977:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573874.816980:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573874.816983:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573874.816996:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573874.817000:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573874.817006:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573875.328919:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573875.328931:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573875.328935:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573875.328939:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573875.328945:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573875.328953:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14562 00000400:00000200:0.0:1788573875.328963:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573875.328970:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573875.328973:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573875.328977:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.328997:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573875.329000:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329003:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573875.329007:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573875.329009:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573875.329012:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573875.329015:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573875.329018:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573875.329020:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573875.329023:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329025:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573875.329027:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329029:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573875.329032:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573875.329035:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329038:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573875.329041:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573875.329044:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573875.329047:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.329049:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573875.329052:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573875.329059:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573875.329063:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573875.329067:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573875.329068:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.329072:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573875.329075:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573875.329079:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573875.329087:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573875.329094:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573875.329104:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516627072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573875.329127:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573875.329131:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573875.329137:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516627072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573875.329158:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573875.329161:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573875.329164:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573875.329166:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573875.329168:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573875.329172:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573875.329175:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14562 00000400:00000200:0.0:1788573875.329180:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573875.329185:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573875.329188:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573875.329192:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329194:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573875.329198:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329204:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573875.329209:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573875.329212:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573875.329215:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573875.329218:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573875.329224:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573875.329230:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573875.329238:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329245:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573875.329250:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329257:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573875.329270:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573875.329279:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329284:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573875.329290:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573875.329298:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573875.329304:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.329309:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573875.329316:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573875.329322:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.329327:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573875.329334:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573875.329340:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573875.329346:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573875.329350:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.329354:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573875.329358:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573875.329363:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573875.329371:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573875.329378:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573875.329388:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516627200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573875.329401:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573875.329407:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573875.329416:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516627200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573875.329447:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573875.329450:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573875.329453:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573875.329457:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573875.329462:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573875.329467:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573875.329480:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14562 00000400:00000200:0.0:1788573875.329489:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573875.329496:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573875.329500:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573875.329508:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329514:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573875.329520:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329525:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573875.329531:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573875.329535:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573875.329539:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573875.329543:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573875.329548:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573875.329553:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573875.329562:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329567:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573875.329573:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329578:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573875.329584:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573875.329590:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329595:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573875.329600:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573875.329606:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573875.329611:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.329616:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573875.329621:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573875.329627:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.329630:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573875.329635:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573875.329641:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573875.329649:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573875.329657:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.329662:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573875.329664:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573875.329669:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573875.329675:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573875.329679:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573875.329686:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516627328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573875.329700:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573875.329706:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573875.329714:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516627328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573875.329734:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573875.329739:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573875.329746:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573875.329750:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573875.329754:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573875.329759:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573875.329765:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14562 00000400:00000200:0.0:1788573875.329776:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573875.329785:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573875.329790:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573875.329797:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329803:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573875.329842:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329847:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573875.329853:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573875.329856:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573875.329861:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573875.329865:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573875.329872:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573875.329887:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573875.329896:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329902:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573875.329909:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329913:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573875.329918:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573875.329926:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573875.329933:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573875.329937:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573875.329943:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573875.329947:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.329952:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573875.329956:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573875.329960:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.329964:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573875.329970:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573875.329975:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573875.329981:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573875.329985:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.329988:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573875.329991:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573875.329994:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573875.330001:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573875.330008:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573875.330019:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516627456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573875.330037:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573875.330041:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573875.330051:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516627456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573875.330077:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573875.330089:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573875.330097:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573875.330103:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573875.330108:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573875.330114:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573875.330123:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14562 00000400:00000200:0.0:1788573875.330134:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573875.330143:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573875.330149:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573875.330157:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.330163:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573875.330170:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.330174:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573875.330179:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573875.330181:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573875.330185:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573875.330188:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573875.330192:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573875.330197:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573875.330205:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.330210:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573875.330217:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573875.330222:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573875.330227:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573875.330235:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573875.330242:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573875.330245:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573875.330248:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573875.330254:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.330259:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573875.330271:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573875.330278:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.330283:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573875.330291:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573875.330297:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573875.330305:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573875.330308:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573875.330313:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573875.330317:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573875.330321:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573875.330328:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573875.330334:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573875.330346:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516627584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573875.330375:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573875.330380:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573875.330392:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516627584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573875.330412:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573875.330419:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14572) 00000100:00000001:0.0:1788573875.840917:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573875.840927:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573875.840932:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573875.840940:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573875.840947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573875.840968:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573875.840987:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573875.840992:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573875.840997:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573876.545025:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573876.545029:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573876.545032:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573876.864964:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573876.864971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573876.864992:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573876.864998:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573876.865011:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573876.865018:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573876.865037:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573876.865041:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573876.865045:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.888946:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573877.888956:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573877.888959:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573877.888972:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573877.888978:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573877.888990:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573877.889002:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573877.889007:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573877.889016:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573877.889061:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573877.889067:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573877.889070:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573877.889076:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573877.889079:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573877.889083:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573877.889086:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573877.889088:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573877.889090:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573878.912924:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573878.912930:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573878.912933:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573878.912940:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573878.912947:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573878.912952:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573878.912959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573878.912962:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573878.912965:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573878.912984:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573878.912991:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573878.912996:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573878.913003:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573878.913017:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573878.913020:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573878.913023:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573878.913026:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573878.913028:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573879.168929:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309239021, at 4309239040 00000100:00000010:0.0:1788573879.168955:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7de0. 00000020:00000001:0.0:1788573879.617071:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573879.617077:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573879.617081:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573879.936907:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573879.936915:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573879.936919:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573879.936927:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573879.936934:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573879.936942:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573879.936969:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573879.936973:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573879.936977:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573880.449116:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788573880.449193:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573880.449199:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573880.449202:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573880.449211:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516627712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.449224:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573880.449228:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573880.449233:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573880.449237:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573880.449244:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573880.449251:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516627712:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573880.449258:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573880.449263:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573880.449265:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573880.449269:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573880.449283:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573880.449287:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573880.449295:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573880.449303:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573880.449313:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573880.449335:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516627712, portal 25 00000100:00000001:0.0:1788573880.449339:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573880.449342:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788573880.449354:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516627712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.449383:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573880.449390:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573880.449396:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573880.449401:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516627712, offset 0 00000400:00000200:0.0:1788573880.449410:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573880.449419:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573880.449427:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573880.449434:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573880.449440:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573880.449446:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573880.449451:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573880.449457:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546556:546556:256:4294967295] 192.168.202.128@tcp LPNI seq info [546479:546479:8:4294967295] 00000400:00000200:0.0:1788573880.449470:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573880.449488:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573880.449499:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573880.449506:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573880.449516:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573880.449522:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788573880.449558:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573880.449564:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573880.449572:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573880.449588:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573880.449595:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573880.449601:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573880.449609:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573880.449619:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516627712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.449639:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573880.449647:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516627712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.449657:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.449660:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573880.449667:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573880.449675:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573880.449679:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309244141 00000100:00000010:0.0:1788573880.449685:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573880.449694:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573880.449700:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309244141 00000100:00000010:0.0:1788573880.449706:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50de0. 00000100:00000001:0.0:1788573880.449712:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573880.449715:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573880.449718:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.449722:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573880.449729:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573880/real 1788573880] req@ffff8fc4901a7480 x1875440516627712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573880.449743:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573880.449745:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573880.449749:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573880.449754:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573880.449757:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573880.449762:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573880.449767:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516627712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573880.449800:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516627712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.449830:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573880.449834:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.449839:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573880.449843:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573880.449853:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516627712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.449867:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573880.449877:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516627712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.449884:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573880.449888:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.449893:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573880.449895:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.449898:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573880.449901:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573880.449905:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573880.449912:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573880.449918:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573880.449928:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516627712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573880.449950:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516627712:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573880.449961:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.449966:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573880.449973:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516627712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.449995:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573880.449999:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573880.450002:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573880.450007:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573880.450011:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573880.450015:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573880.450019:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573880.450022:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7000. 02000000:00000001:0.0:1788573880.450026:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573880.450028:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573880.450031:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788573880.450035:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573880.450038:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.450041:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.450048:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573880.450051:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.450055:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573880.450058:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573880.450062:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573880.450069:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516627840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450080:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573880.450081:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573880.450085:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573880.450090:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573880.450096:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573880.450105:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516627840:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573880.450112:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573880.450116:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573880.450121:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573880.450126:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573880.450131:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573880.450135:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573880.450141:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573880.450151:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573880.450156:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573880.450162:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516627840, portal 10 00000100:00000001:0.0:1788573880.450167:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573880.450170:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788573880.450180:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516627840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450192:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573880.450196:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573880.450201:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573880.450205:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516627840, offset 0 00000400:00000200:0.0:1788573880.450211:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573880.450217:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573880.450225:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573880.450231:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573880.450236:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573880.450241:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573880.450249:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573880.450254:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546557:546557:256:4294967295] 192.168.202.128@tcp LPNI seq info [546480:546480:8:4294967295] 00000400:00000200:0.0:1788573880.450259:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573880.450267:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573880.450272:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573880.450278:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573880.450289:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573880.450296:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788573880.450345:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573880.450349:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573880.450356:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573880.450365:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573880.450368:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573880.450371:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573880.450381:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573880.450388:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516627840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450399:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573880.450405:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516627840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450424:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.450428:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573880.450435:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573880.450440:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573880.450445:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573880.450448:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309240339 00000100:00000001:0.0:1788573880.450452:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309244141 00000100:00000001:0.0:1788573880.450455:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309240339 00000100:00000001:0.0:1788573880.450458:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309244141 00000100:00000001:0.0:1788573880.450461:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573880.450464:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.450466:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573880.450472:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573880/real 1788573880] req@ffff8fc4901a5880 x1875440516627840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573880.450479:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573880.450481:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573880.450483:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573880.450486:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573880.450488:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573880.450491:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573880.450494:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516627840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573880.450500:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516627840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450513:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573880.450514:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.450516:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573880.450517:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573880.450519:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516627840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450524:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573880.450527:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516627840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450531:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573880.450533:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.450534:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573880.450535:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.450536:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573880.450537:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573880.450539:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573880.450541:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573880.450542:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573880.450545:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516627840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573880.450550:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516627840:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573880.450553:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573880.450555:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516627968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450559:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573880.450560:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573880.450564:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573880.450566:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573880.450567:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573880.450569:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516627968:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573880.450572:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573880.450573:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573880.450574:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573880.450576:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573880.450578:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573880.450579:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573880.450582:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573880.450584:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573880.450586:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573880.450588:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516627968, portal 10 00000100:00000001:0.0:1788573880.450590:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573880.450591:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788573880.450596:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516627968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450603:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573880.450606:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573880.450608:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573880.450610:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516627968, offset 0 00000400:00000200:0.0:1788573880.450614:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573880.450619:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573880.450623:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573880.450626:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573880.450629:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573880.450632:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573880.450634:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573880.450638:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546558:546558:256:4294967295] 192.168.202.128@tcp LPNI seq info [546481:546481:8:4294967295] 00000400:00000200:0.0:1788573880.450642:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573880.450649:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573880.450659:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573880.450662:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573880.450664:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573880.450666:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788573880.450674:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573880.450675:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573880.450677:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573880.450680:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573880.450682:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573880.450684:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573880.450686:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573880.450689:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516627968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450694:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573880.450696:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516627968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450701:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.450702:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573880.450704:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573880.450706:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573880.450708:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573880.450710:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309240346 00000100:00000001:0.0:1788573880.450712:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309244141 00000100:00000001:0.0:1788573880.450713:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309240346 00000100:00000001:0.0:1788573880.450715:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309244141 00000100:00000001:0.0:1788573880.450716:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573880.450718:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.450719:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573880.450721:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573880/real 1788573880] req@ffff8fc4901a4700 x1875440516627968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573880.450730:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573880.450731:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573880.450733:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573880.450734:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573880.450736:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573880.450737:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573880.450739:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516627968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573880.450745:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516627968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450752:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573880.450754:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.450756:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573880.450757:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573880.450761:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516627968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450769:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573880.450773:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516627968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450780:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573880.450783:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.450785:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573880.450786:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.450788:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573880.450790:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573880.450793:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573880.450797:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573880.450799:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573880.450830:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516627968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573880.450839:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516627968:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573880.450844:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.450846:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573880.450850:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516627840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450856:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573880.450858:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573880.450860:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573880.450863:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573880.450865:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573880.450866:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573880.450868:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573880.450870:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6400. 02000000:00000001:0.0:1788573880.450873:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573880.450874:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573880.450875:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788573880.450880:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573880.450881:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.450883:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573880.450886:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516627968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450893:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573880.450895:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573880.450896:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573880.450900:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573880.450901:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573880.450902:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573880.450904:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573880.450906:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5c00. 02000000:00000001:0.0:1788573880.450908:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573880.450915:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573880.450917:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788573880.450920:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573880.450922:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.450924:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.450928:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573880.450930:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.450932:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573880.450933:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573880.450935:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573880.450939:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516628096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.450947:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573880.450948:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573880.450951:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573880.450953:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573880.450955:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573880.450958:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516628096:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573880.450961:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573880.450963:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573880.450965:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573880.450967:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573880.450971:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573880.450973:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573880.450977:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573880.450980:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573880.450984:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573880.450987:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516628096, portal 4 00000100:00000001:0.0:1788573880.450989:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573880.450990:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788573880.450996:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516628096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451002:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573880.451004:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573880.451012:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573880.451014:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516628096, offset 0 00000400:00000200:0.0:1788573880.451019:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573880.451023:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573880.451027:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573880.451030:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573880.451032:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573880.451035:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573880.451037:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573880.451040:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546559:546559:256:4294967295] 192.168.202.128@tcp LPNI seq info [546482:546482:8:4294967295] 00000400:00000200:0.0:1788573880.451044:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573880.451050:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573880.451055:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573880.451059:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573880.451063:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573880.451065:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788573880.451076:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573880.451078:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573880.451081:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573880.451084:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573880.451087:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573880.451089:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573880.451092:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573880.451096:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516628096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451104:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573880.451107:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516628096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451114:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.451115:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573880.451125:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573880.451128:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573880.451132:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573880.451134:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309240487 00000100:00000001:0.0:1788573880.451137:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309244141 00000100:00000001:0.0:1788573880.451139:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309240487 00000100:00000001:0.0:1788573880.451142:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309244141 00000100:00000001:0.0:1788573880.451144:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573880.451146:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.451147:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573880.451152:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573880/real 1788573880] req@ffff8fc4901a4e00 x1875440516628096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573880.451161:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573880.451163:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573880.451165:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573880.451168:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573880.451170:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573880.451172:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573880.451175:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516628096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573880.451184:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516628096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451190:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573880.451192:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.451194:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573880.451195:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573880.451199:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516628096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451206:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573880.451211:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516628096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451225:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573880.451228:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.451230:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573880.451231:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.451233:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573880.451235:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573880.451237:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573880.451241:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573880.451244:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573880.451248:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516628096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573880.451256:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516628096:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573880.451260:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573880.451263:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516628224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451270:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573880.451272:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573880.451274:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573880.451276:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573880.451278:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573880.451281:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516628224:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573880.451285:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573880.451287:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573880.451289:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573880.451290:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573880.451294:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573880.451296:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573880.451301:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573880.451310:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573880.451314:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573880.451318:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516628224, portal 4 00000100:00000001:0.0:1788573880.451320:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573880.451334:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788573880.451342:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516628224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451348:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573880.451351:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573880.451354:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573880.451357:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516628224, offset 0 00000400:00000200:0.0:1788573880.451361:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573880.451366:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573880.451370:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573880.451373:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573880.451376:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573880.451379:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573880.451382:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573880.451385:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546560:546560:256:4294967295] 192.168.202.128@tcp LPNI seq info [546483:546483:8:4294967295] 00000400:00000200:0.0:1788573880.451389:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573880.451396:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573880.451401:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573880.451404:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573880.451408:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573880.451410:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788573880.451422:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573880.451424:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573880.451427:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573880.451430:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573880.451434:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573880.451436:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573880.451446:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573880.451450:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516628224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451458:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573880.451461:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516628224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573905 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451466:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.451468:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573880.451471:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573880.451474:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573880.451477:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573880.451479:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309240489 00000100:00000001:0.0:1788573880.451481:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309244141 00000100:00000001:0.0:1788573880.451484:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309240489 00000100:00000001:0.0:1788573880.451486:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309244141 00000100:00000001:0.0:1788573880.451488:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573880.451490:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.451491:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573880.451495:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573880/real 1788573880] req@ffff8fc4901a6300 x1875440516628224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573880.451503:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573880.451504:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573880.451506:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573880.451508:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573880.451510:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573880.451512:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573880.451515:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516628224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573880.451522:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516628224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451538:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573880.451540:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.451542:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573880.451543:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573880.451548:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516628224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451553:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573880.451558:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516628224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451564:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573880.451566:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.451567:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573880.451569:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.451571:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573880.451573:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573880.451575:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573880.451578:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573880.451580:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573880.451584:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516628224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573880.451591:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516628224:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573880.451595:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.451597:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573880.451600:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516628096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451606:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573880.451608:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573880.451614:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573880.451617:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573880.451618:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573880.451620:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573880.451621:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573880.451623:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788573880.451626:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573880.451627:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573880.451628:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788573880.451632:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573880.451633:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.451634:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573880.451638:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516628224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573905 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573880.451645:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573880.451646:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573880.451647:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573880.451649:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573880.451650:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573880.451651:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573880.451653:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573880.451654:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788573880.451657:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573880.451658:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573880.451659:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788573880.451663:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573880.451665:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.451667:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.451671:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573880.451672:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.451674:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573880.451676:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573880.960921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573880.960930:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573880.960934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573880.960944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573880.960951:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573880.960970:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.960977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573880.960980:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573880.960985:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573881.665040:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573881.665044:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573881.665048:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573881.984871:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573881.984879:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573881.984881:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573881.984885:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573881.984891:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573881.984897:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573881.984903:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573881.984908:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573881.984915:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573881.984962:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573881.984968:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573881.984976:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573881.984985:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573881.984989:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573881.984991:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573881.984997:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573881.984999:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573881.985001:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573883.008908:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573883.008918:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573883.008922:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573883.008934:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573883.008942:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573883.008948:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573883.008963:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573883.008966:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573883.008968:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573884.032927:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573884.032930:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573884.032933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573884.032937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573884.032954:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573884.032961:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573884.032965:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573884.032967:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573884.032970:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573884.288894:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309244141, at 4309244160 00000100:00000010:0.0:1788573884.288912:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573884.736969:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573884.736971:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573884.736973:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573885.056918:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573885.056928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573885.056933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573885.056942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573885.056951:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573885.056956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.056971:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573885.056974:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.056977:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.569210:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573885.569216:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573885.569243:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573885.569251:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5180 x1875440516628352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.569262:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573885.569265:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573885.569269:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573885.569272:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573885.569275:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573885.569281:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516628352:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573885.569285:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573885.569292:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573885.569298:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573885.569304:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573885.569312:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573885.569325:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573885.569336:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573885.569344:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573885.569354:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573885.569362:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516628352, portal 25 00000100:00000001:0.0:1788573885.569367:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573885.569369:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605325184 : -123400104226432 : ffff8fc4ac9f5180) 00000100:00000040:0.0:1788573885.569380:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5180 x1875440516628352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.569388:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573885.569396:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573885.569400:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573885.569406:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516628352, offset 0 00000400:00000200:0.0:1788573885.569416:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573885.569428:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573885.569435:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573885.569443:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573885.569450:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573885.569458:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573885.569462:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573885.569468:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546561:546561:256:4294967295] 192.168.202.128@tcp LPNI seq info [546484:546484:8:4294967295] 00000400:00000200:0.0:1788573885.569478:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573885.569492:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573885.569503:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573885.569509:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573885.569518:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573885.569523:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788573885.569542:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573885.569545:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573885.569551:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573885.569560:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573885.569577:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573885.569580:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573885.569588:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573885.569596:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5180 x1875440516628352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.569612:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573885.569617:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5180 x1875440516628352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.569632:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.569635:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573885.569642:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573885.569649:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573885.569652:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309249261 00000100:00000010:0.0:1788573885.569656:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573885.569663:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573885.569671:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309249261 00000100:00000010:0.0:1788573885.569677:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50e40. 00000100:00000001:0.0:1788573885.569683:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573885.569687:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573885.569690:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.569696:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573885.569702:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573885/real 1788573885] req@ffff8fc4ac9f5180 x1875440516628352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573885.569717:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573885.569720:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573885.569723:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573885.569729:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573885.569732:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573885.569735:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573885.569740:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516628352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573885.569754:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516628352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.569781:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573885.569785:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.569789:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573885.569792:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573885.569801:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516628352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.569832:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573885.569841:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516628352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.569855:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573885.569861:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.569864:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573885.569867:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.569870:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573885.569874:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573885.569880:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573885.569886:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573885.569891:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573885.569899:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516628352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573885.569917:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516628352:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573885.569923:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.569927:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573885.569929:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516628352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.569946:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573885.569949:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573885.569959:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573885.569965:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573885.569969:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573885.569971:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573885.569974:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573885.569977:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36400. 02000000:00000001:0.0:1788573885.569982:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573885.569985:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573885.569988:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788573885.569991:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573885.569992:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.569995:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570001:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573885.570004:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.570008:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573885.570012:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573885.570015:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573885.570021:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4e00 x1875440516628480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570032:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573885.570035:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573885.570037:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573885.570041:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573885.570042:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573885.570045:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516628480:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573885.570047:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573885.570049:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573885.570050:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573885.570051:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573885.570054:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573885.570055:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573885.570059:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573885.570061:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573885.570063:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573885.570066:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516628480, portal 10 00000100:00000001:0.0:1788573885.570070:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573885.570071:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605324288 : -123400104227328 : ffff8fc4ac9f4e00) 00000100:00000040:0.0:1788573885.570076:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4e00 x1875440516628480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570081:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573885.570082:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573885.570084:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573885.570085:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516628480, offset 0 00000400:00000200:0.0:1788573885.570088:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573885.570091:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573885.570094:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573885.570096:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573885.570098:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573885.570100:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573885.570101:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573885.570103:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546562:546562:256:4294967295] 192.168.202.128@tcp LPNI seq info [546485:546485:8:4294967295] 00000400:00000200:0.0:1788573885.570106:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573885.570111:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573885.570114:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573885.570117:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573885.570119:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573885.570121:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788573885.570128:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573885.570130:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573885.570131:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573885.570134:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573885.570136:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573885.570137:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573885.570139:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573885.570141:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4e00 x1875440516628480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570157:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573885.570160:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4e00 x1875440516628480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570167:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.570168:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573885.570172:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573885.570178:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573885.570182:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573885.570184:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309245459 00000100:00000001:0.0:1788573885.570186:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309249261 00000100:00000001:0.0:1788573885.570188:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309245459 00000100:00000001:0.0:1788573885.570190:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309249261 00000100:00000001:0.0:1788573885.570192:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573885.570195:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.570196:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573885.570200:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573885/real 1788573885] req@ffff8fc4ac9f4e00 x1875440516628480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573885.570208:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573885.570209:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573885.570211:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573885.570212:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573885.570214:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573885.570216:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573885.570235:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4e00 x1875440516628480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573885.570243:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4e00 x1875440516628480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570249:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573885.570256:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570258:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573885.570260:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573885.570263:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4e00 x1875440516628480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570270:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573885.570274:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4e00 x1875440516628480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570281:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573885.570284:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570286:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573885.570287:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570289:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573885.570291:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573885.570294:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573885.570297:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573885.570299:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573885.570303:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4e00 x1875440516628480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573885.570310:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516628480:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573885.570314:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573885.570317:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516628608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570323:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573885.570326:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573885.570327:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573885.570329:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573885.570331:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573885.570340:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516628608:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573885.570342:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573885.570344:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573885.570346:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573885.570348:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573885.570351:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573885.570353:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573885.570356:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573885.570358:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573885.570362:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573885.570365:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516628608, portal 10 00000100:00000001:0.0:1788573885.570366:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573885.570368:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788573885.570374:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516628608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570379:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573885.570381:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573885.570383:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573885.570386:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516628608, offset 0 00000400:00000200:0.0:1788573885.570389:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573885.570393:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573885.570397:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573885.570399:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573885.570402:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573885.570404:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573885.570406:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573885.570409:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546563:546563:256:4294967295] 192.168.202.128@tcp LPNI seq info [546486:546486:8:4294967295] 00000400:00000200:0.0:1788573885.570413:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573885.570420:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573885.570424:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573885.570427:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573885.570438:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573885.570441:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788573885.570448:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573885.570449:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573885.570451:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573885.570454:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573885.570457:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573885.570458:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573885.570460:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573885.570463:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516628608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570468:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573885.570470:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516628608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570475:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.570476:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573885.570478:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573885.570481:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573885.570483:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573885.570484:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309245466 00000100:00000001:0.0:1788573885.570486:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309249261 00000100:00000001:0.0:1788573885.570487:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309245466 00000100:00000001:0.0:1788573885.570489:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309249261 00000100:00000001:0.0:1788573885.570490:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573885.570492:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.570493:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573885.570495:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573885/real 1788573885] req@ffff8fc4ac9f6a00 x1875440516628608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573885.570500:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573885.570504:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573885.570506:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573885.570507:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573885.570508:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573885.570510:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573885.570512:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516628608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573885.570518:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516628608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570522:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573885.570523:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570524:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573885.570525:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573885.570527:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516628608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570531:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573885.570534:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516628608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570539:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573885.570540:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570541:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573885.570542:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570543:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573885.570545:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573885.570546:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573885.570548:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573885.570550:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573885.570552:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516628608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573885.570563:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516628608:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573885.570566:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573885.570568:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6680 x1875440516628736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570572:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573885.570573:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573885.570575:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573885.570576:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573885.570577:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573885.570579:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516628736:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573885.570581:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573885.570583:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573885.570584:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573885.570585:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573885.570587:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573885.570588:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573885.570591:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573885.570593:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573885.570595:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573885.570597:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516628736, portal 4 00000100:00000001:0.0:1788573885.570598:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573885.570600:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605330560 : -123400104221056 : ffff8fc4ac9f6680) 00000100:00000040:0.0:1788573885.570604:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6680 x1875440516628736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570608:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573885.570610:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573885.570611:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573885.570612:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516628736, offset 0 00000400:00000200:0.0:1788573885.570615:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573885.570621:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573885.570623:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573885.570625:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573885.570627:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573885.570629:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573885.570630:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573885.570632:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546564:546564:256:4294967295] 192.168.202.128@tcp LPNI seq info [546487:546487:8:4294967295] 00000400:00000200:0.0:1788573885.570635:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573885.570639:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573885.570642:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573885.570644:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573885.570646:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573885.570648:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788573885.570655:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573885.570656:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573885.570658:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573885.570660:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573885.570663:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573885.570664:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573885.570666:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573885.570668:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6680 x1875440516628736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570673:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573885.570675:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6680 x1875440516628736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570680:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.570681:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573885.570682:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573885.570684:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573885.570686:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573885.570691:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309245606 00000100:00000001:0.0:1788573885.570693:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309249261 00000100:00000001:0.0:1788573885.570695:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309245606 00000100:00000001:0.0:1788573885.570696:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309249261 00000100:00000001:0.0:1788573885.570698:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573885.570699:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.570700:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573885.570703:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573885/real 1788573885] req@ffff8fc4ac9f6680 x1875440516628736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573885.570708:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573885.570709:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573885.570710:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573885.570711:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573885.570712:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573885.570713:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573885.570715:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6680 x1875440516628736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573885.570721:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6680 x1875440516628736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570725:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573885.570726:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570727:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573885.570727:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573885.570730:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6680 x1875440516628736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570733:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573885.570736:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6680 x1875440516628736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570740:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573885.570744:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570745:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573885.570746:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570747:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573885.570749:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573885.570750:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573885.570752:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573885.570754:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573885.570756:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6680 x1875440516628736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573885.570761:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516628736:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573885.570763:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570765:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573885.570767:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4e00 x1875440516628480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570771:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573885.570772:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573885.570773:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573885.570775:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573885.570776:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573885.570776:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573885.570778:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573885.570779:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37c00. 02000000:00000001:0.0:1788573885.570781:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573885.570782:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573885.570783:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4e00. 00000100:00000001:0.0:1788573885.570785:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573885.570786:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570787:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573885.570788:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516628608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570799:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573885.570800:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573885.570801:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573885.570804:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573885.570805:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573885.570805:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573885.570806:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573885.570822:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35400. 02000000:00000001:0.0:1788573885.570825:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573885.570825:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573885.570826:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788573885.570828:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573885.570829:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570830:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573885.570833:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6680 x1875440516628736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570837:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573885.570838:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573885.570839:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573885.570841:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573885.570841:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573885.570842:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573885.570844:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573885.570845:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35c00. 02000000:00000001:0.0:1788573885.570846:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573885.570846:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573885.570847:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6680. 00000100:00000001:0.0:1788573885.570848:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573885.570849:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570850:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.570853:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573885.570853:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.570855:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573885.570856:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573885.570857:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573885.570864:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7480 x1875440516628864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570868:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573885.570870:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573885.570871:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573885.570872:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573885.570874:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573885.570875:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516628864:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573885.570877:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573885.570878:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573885.570879:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573885.570880:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573885.570882:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573885.570883:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573885.570886:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573885.570887:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573885.570890:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573885.570892:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516628864, portal 4 00000100:00000001:0.0:1788573885.570893:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573885.570894:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605334144 : -123400104217472 : ffff8fc4ac9f7480) 00000100:00000040:0.0:1788573885.570898:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7480 x1875440516628864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570902:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573885.570904:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573885.570905:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573885.570907:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516628864, offset 0 00000400:00000200:0.0:1788573885.570909:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573885.570911:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573885.570914:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573885.570915:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573885.570917:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573885.570919:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573885.570921:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573885.570926:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546565:546565:256:4294967295] 192.168.202.128@tcp LPNI seq info [546488:546488:8:4294967295] 00000400:00000200:0.0:1788573885.570929:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573885.570933:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573885.570936:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573885.570938:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573885.570941:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573885.570943:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788573885.570949:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573885.570951:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573885.570953:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573885.570955:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573885.570957:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573885.570958:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573885.570960:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573885.570962:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7480 x1875440516628864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570968:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573885.570970:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7480 x1875440516628864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573910 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.570974:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.570975:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573885.570977:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573885.570979:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573885.570981:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573885.570982:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309245609 00000100:00000001:0.0:1788573885.570984:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309249261 00000100:00000001:0.0:1788573885.570986:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309245609 00000100:00000001:0.0:1788573885.570988:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309249261 00000100:00000001:0.0:1788573885.570989:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573885.570993:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.570995:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573885.570997:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573885/real 1788573885] req@ffff8fc4ac9f7480 x1875440516628864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573885.571003:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573885.571004:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573885.571005:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573885.571007:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573885.571008:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573885.571009:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573885.571011:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7480 x1875440516628864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573885.571016:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7480 x1875440516628864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.571020:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573885.571021:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.571023:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573885.571023:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573885.571025:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7480 x1875440516628864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.571029:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573885.571032:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7480 x1875440516628864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.571036:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573885.571037:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.571038:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573885.571039:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.571040:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573885.571041:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573885.571043:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573885.571048:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573885.571049:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573885.571052:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7480 x1875440516628864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573885.571057:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516628864:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573885.571059:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.571060:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573885.571062:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7480 x1875440516628864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573910 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573885.571066:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573885.571067:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573885.571068:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573885.571069:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573885.571070:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573885.571071:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573885.571072:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573885.571073:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34c00. 02000000:00000001:0.0:1788573885.571075:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573885.571076:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573885.571077:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7480. 00000100:00000001:0.0:1788573885.571079:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573885.571080:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.571081:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573885.571085:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573885.571086:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573885.571088:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573885.571089:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573886.080935:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573886.080939:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573886.080943:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573886.080951:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573886.080958:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573886.080963:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573886.080986:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573886.080990:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573886.080996:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573886.785005:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573886.785008:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573886.785011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573887.104910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573887.104920:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573887.104924:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573887.104933:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573887.104941:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573887.104946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573887.104951:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573887.104955:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573887.104959:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573888.128947:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573888.128953:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573888.128960:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573888.128968:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573888.128974:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573888.128977:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573888.129009:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573888.129013:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573888.129017:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573889.152908:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573889.152915:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573889.152918:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573889.152925:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573889.152931:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573889.152937:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573889.152975:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573889.152979:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573889.152983:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573889.408924:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309249261, at 4309249280 00000100:00000010:0.0:1788573889.408940:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573889.856988:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573889.856992:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573889.857013:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573890.176893:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573890.176904:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573890.176908:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573890.176914:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573890.176922:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573890.176926:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.176939:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573890.176944:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.176948:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.176967:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573890.176972:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573890.176974:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573890.176977:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573890.176980:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573890.176983:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.176986:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573890.176988:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.176991:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.689176:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573890.689182:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573890.689186:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573890.689197:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7800 x1875440516628992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.689218:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573890.689222:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573890.689229:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573890.689233:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573890.689240:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573890.689248:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516628992:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573890.689255:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573890.689259:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573890.689263:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573890.689268:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573890.689275:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573890.689280:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573890.689316:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573890.689324:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573890.689335:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573890.689344:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516628992, portal 25 00000100:00000001:0.0:1788573890.689350:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573890.689355:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335040 : -123400104216576 : ffff8fc4ac9f7800) 00000100:00000040:0.0:1788573890.689365:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7800 x1875440516628992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.689374:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573890.689379:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573890.689383:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573890.689387:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516628992, offset 0 00000400:00000200:0.0:1788573890.689398:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573890.689408:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573890.689416:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573890.689422:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573890.689427:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573890.689433:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573890.689436:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573890.689442:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546566:546566:256:4294967295] 192.168.202.128@tcp LPNI seq info [546489:546489:8:4294967295] 00000400:00000200:0.0:1788573890.689450:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573890.689464:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573890.689474:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573890.689480:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573890.689489:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573890.689495:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573890.689523:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573890.689526:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573890.689534:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573890.689541:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573890.689547:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573890.689557:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573890.689565:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573890.689573:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7800 x1875440516628992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.689589:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573890.689596:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7800 x1875440516628992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.689607:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.689610:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573890.689616:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573890.689622:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573890.689625:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309254381 00000100:00000010:0.0:1788573890.689632:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573890.689640:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573890.689646:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309254381 00000100:00000010:0.0:1788573890.689652:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50300. 00000100:00000001:0.0:1788573890.689656:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573890.689660:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573890.689664:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.689669:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573890.689675:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573890/real 1788573890] req@ffff8fc4ac9f7800 x1875440516628992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573890.689686:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573890.689690:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573890.689693:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573890.689699:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573890.689701:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573890.689706:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573890.689714:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7800 x1875440516628992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573890.689727:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7800 x1875440516628992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.689752:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573890.689757:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.689763:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573890.689769:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573890.689782:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7800 x1875440516628992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.689805:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573890.689846:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7800 x1875440516628992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.689857:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573890.689862:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.689865:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573890.689868:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.689884:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573890.689887:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573890.689890:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573890.689896:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573890.689899:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573890.689909:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7800 x1875440516628992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573890.689925:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516628992:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573890.689932:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.689938:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573890.689944:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7800 x1875440516628992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.689953:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573890.689957:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573890.689966:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573890.689970:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573890.689973:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573890.689976:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573890.689978:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573890.689981:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35000. 02000000:00000001:0.0:1788573890.689987:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573890.689989:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573890.689992:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7800. 00000100:00000001:0.0:1788573890.689996:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573890.689998:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.690003:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.690010:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573890.690013:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.690015:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573890.690017:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573890.690020:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573890.690029:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4700 x1875440516629120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690048:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573890.690051:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573890.690054:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573890.690057:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573890.690063:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573890.690079:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516629120:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573890.690089:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573890.690092:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573890.690096:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573890.690100:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573890.690107:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573890.690110:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573890.690113:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573890.690129:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573890.690135:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573890.690140:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516629120, portal 10 00000100:00000001:0.0:1788573890.690149:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573890.690153:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605322496 : -123400104229120 : ffff8fc4ac9f4700) 00000100:00000040:0.0:1788573890.690162:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4700 x1875440516629120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690172:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573890.690178:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573890.690182:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573890.690185:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516629120, offset 0 00000400:00000200:0.0:1788573890.690191:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573890.690201:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573890.690210:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573890.690217:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573890.690224:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573890.690232:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573890.690239:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573890.690247:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546567:546567:256:4294967295] 192.168.202.128@tcp LPNI seq info [546490:546490:8:4294967295] 00000400:00000200:0.0:1788573890.690253:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573890.690263:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573890.690269:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573890.690274:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573890.690284:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573890.690289:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573890.690312:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573890.690316:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573890.690322:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573890.690330:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573890.690334:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573890.690339:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573890.690345:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573890.690351:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4700 x1875440516629120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690391:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573890.690399:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4700 x1875440516629120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690409:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.690410:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573890.690414:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573890.690417:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573890.690420:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573890.690422:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309250579 00000100:00000001:0.0:1788573890.690424:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309254381 00000100:00000001:0.0:1788573890.690426:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309250579 00000100:00000001:0.0:1788573890.690427:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309254381 00000100:00000001:0.0:1788573890.690429:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573890.690430:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.690431:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573890.690434:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573890/real 1788573890] req@ffff8fc4ac9f4700 x1875440516629120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573890.690439:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573890.690440:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573890.690442:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573890.690443:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573890.690444:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573890.690446:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573890.690448:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4700 x1875440516629120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573890.690453:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4700 x1875440516629120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690457:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573890.690461:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.690462:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573890.690463:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573890.690466:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4700 x1875440516629120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690471:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573890.690473:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4700 x1875440516629120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690477:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573890.690479:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.690480:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573890.690481:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.690482:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573890.690484:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573890.690486:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573890.690488:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573890.690489:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573890.690491:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4700 x1875440516629120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573890.690497:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516629120:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573890.690500:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573890.690502:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5f80 x1875440516629248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690506:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573890.690507:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573890.690509:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573890.690510:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573890.690511:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573890.690517:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516629248:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573890.690520:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573890.690521:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573890.690522:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573890.690524:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573890.690526:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573890.690527:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573890.690530:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573890.690531:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573890.690533:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573890.690536:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516629248, portal 10 00000100:00000001:0.0:1788573890.690537:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573890.690538:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605328768 : -123400104222848 : ffff8fc4ac9f5f80) 00000100:00000040:0.0:1788573890.690542:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5f80 x1875440516629248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690546:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573890.690548:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573890.690549:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573890.690551:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516629248, offset 0 00000400:00000200:0.0:1788573890.690553:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573890.690556:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573890.690559:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573890.690561:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573890.690563:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573890.690565:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573890.690566:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573890.690569:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546568:546568:256:4294967295] 192.168.202.128@tcp LPNI seq info [546491:546491:8:4294967295] 00000400:00000200:0.0:1788573890.690571:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573890.690576:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573890.690579:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573890.690581:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573890.690587:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573890.690589:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573890.690597:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573890.690598:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573890.690600:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573890.690603:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573890.690605:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573890.690606:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573890.690608:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573890.690610:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5f80 x1875440516629248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690616:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573890.690618:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5f80 x1875440516629248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690622:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.690623:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573890.690625:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573890.690627:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573890.690629:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573890.690630:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309250586 00000100:00000001:0.0:1788573890.690632:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309254381 00000100:00000001:0.0:1788573890.690633:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309250586 00000100:00000001:0.0:1788573890.690635:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309254381 00000100:00000001:0.0:1788573890.690636:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573890.690638:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.690639:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573890.690641:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573890/real 1788573890] req@ffff8fc4ac9f5f80 x1875440516629248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573890.690646:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573890.690650:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573890.690651:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573890.690652:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573890.690654:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573890.690655:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573890.690658:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5f80 x1875440516629248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573890.690663:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5f80 x1875440516629248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690667:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573890.690668:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.690669:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573890.690670:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573890.690672:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5f80 x1875440516629248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690676:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573890.690679:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5f80 x1875440516629248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690683:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573890.690684:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.690685:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573890.690686:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.690687:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573890.690689:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573890.690691:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573890.690694:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573890.690696:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573890.690700:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5f80 x1875440516629248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573890.690719:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516629248:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573890.690724:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.690726:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573890.690729:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4700 x1875440516629120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690737:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573890.690739:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573890.690741:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573890.690744:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573890.690746:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573890.690747:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573890.690749:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573890.690751:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35800. 02000000:00000001:0.0:1788573890.690752:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573890.690753:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573890.690754:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4700. 00000100:00000001:0.0:1788573890.690756:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573890.690757:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.690758:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573890.690761:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5f80 x1875440516629248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690765:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573890.690766:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573890.690767:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573890.690770:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573890.690771:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573890.690771:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573890.690773:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573890.690773:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788573890.690775:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573890.690775:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573890.690776:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5f80. 00000100:00000001:0.0:1788573890.690782:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573890.690783:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.690784:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.690787:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573890.690788:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.690789:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573890.690791:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573890.690792:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573890.690794:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7480 x1875440516629376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690799:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573890.690800:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573890.690801:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573890.690803:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573890.690804:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573890.690806:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516629376:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573890.690824:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573890.690826:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573890.690827:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573890.690828:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573890.690832:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573890.690834:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573890.690838:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573890.690840:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573890.690845:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573890.690847:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516629376, portal 4 00000100:00000001:0.0:1788573890.690849:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573890.690851:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605334144 : -123400104217472 : ffff8fc4ac9f7480) 00000100:00000040:0.0:1788573890.690857:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7480 x1875440516629376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690864:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573890.690867:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573890.690869:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573890.690872:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516629376, offset 0 00000400:00000200:0.0:1788573890.690882:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573890.690887:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573890.690889:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573890.690891:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573890.690894:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573890.690895:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573890.690897:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573890.690899:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546569:546569:256:4294967295] 192.168.202.128@tcp LPNI seq info [546492:546492:8:4294967295] 00000400:00000200:0.0:1788573890.690902:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573890.690907:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573890.690910:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573890.690913:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573890.690915:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573890.690917:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573890.690928:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573890.690930:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573890.690932:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573890.690934:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573890.690937:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573890.690938:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573890.690940:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573890.690943:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7480 x1875440516629376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690948:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573890.690950:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7480 x1875440516629376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.690955:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.690956:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573890.690958:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573890.690960:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573890.690963:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573890.690968:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309250727 00000100:00000001:0.0:1788573890.690970:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309254381 00000100:00000001:0.0:1788573890.690972:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309250727 00000100:00000001:0.0:1788573890.690973:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309254381 00000100:00000001:0.0:1788573890.690974:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573890.690976:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.690977:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573890.690980:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573890/real 1788573890] req@ffff8fc4ac9f7480 x1875440516629376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573890.690985:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573890.690987:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573890.690988:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573890.690989:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573890.690990:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573890.690992:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573890.690993:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7480 x1875440516629376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573890.690999:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7480 x1875440516629376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.691003:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573890.691004:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.691005:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573890.691006:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573890.691008:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7480 x1875440516629376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.691012:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573890.691014:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7480 x1875440516629376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.691018:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573890.691022:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.691024:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573890.691024:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.691026:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573890.691027:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573890.691029:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573890.691031:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573890.691032:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573890.691035:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7480 x1875440516629376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573890.691040:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516629376:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573890.691043:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573890.691045:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6680 x1875440516629504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.691049:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573890.691050:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573890.691051:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573890.691053:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573890.691055:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573890.691058:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516629504:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573890.691061:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573890.691062:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573890.691064:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573890.691066:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573890.691068:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573890.691070:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573890.691074:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573890.691076:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573890.691080:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573890.691083:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516629504, portal 4 00000100:00000001:0.0:1788573890.691090:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573890.691091:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605330560 : -123400104221056 : ffff8fc4ac9f6680) 00000100:00000040:0.0:1788573890.691098:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6680 x1875440516629504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.691105:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573890.691107:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573890.691109:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573890.691112:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516629504, offset 0 00000400:00000200:0.0:1788573890.691127:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573890.691131:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573890.691134:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573890.691136:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573890.691138:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573890.691140:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573890.691141:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573890.691143:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546570:546570:256:4294967295] 192.168.202.128@tcp LPNI seq info [546493:546493:8:4294967295] 00000400:00000200:0.0:1788573890.691146:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573890.691151:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573890.691154:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573890.691156:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573890.691159:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573890.691161:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573890.691168:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573890.691169:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573890.691172:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573890.691174:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573890.691176:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573890.691177:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573890.691179:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573890.691181:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6680 x1875440516629504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.691197:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573890.691201:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6680 x1875440516629504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573915 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.691207:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.691209:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573890.691212:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573890.691216:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573890.691219:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573890.691222:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309250729 00000100:00000001:0.0:1788573890.691224:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309254381 00000100:00000001:0.0:1788573890.691226:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309250729 00000100:00000001:0.0:1788573890.691229:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309254381 00000100:00000001:0.0:1788573890.691231:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573890.691233:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.691235:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573890.691240:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573890/real 1788573890] req@ffff8fc4ac9f6680 x1875440516629504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573890.691248:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573890.691249:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573890.691251:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573890.691252:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573890.691253:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573890.691255:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573890.691257:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6680 x1875440516629504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573890.691263:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6680 x1875440516629504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.691267:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573890.691273:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.691274:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573890.691275:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573890.691278:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6680 x1875440516629504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.691283:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573890.691285:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6680 x1875440516629504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.691289:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573890.691291:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.691292:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573890.691294:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.691295:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573890.691297:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573890.691300:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573890.691303:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573890.691306:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573890.691310:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6680 x1875440516629504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573890.691319:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516629504:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573890.691323:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.691325:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573890.691329:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7480 x1875440516629376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.691336:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573890.691338:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573890.691341:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573890.691344:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573890.691345:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573890.691352:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573890.691355:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573890.691357:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788573890.691360:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573890.691362:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573890.691363:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7480. 00000100:00000001:0.0:1788573890.691366:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573890.691367:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.691369:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573890.691373:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6680 x1875440516629504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573915 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573890.691378:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573890.691379:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573890.691381:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573890.691382:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573890.691383:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573890.691383:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573890.691385:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573890.691386:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788573890.691387:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573890.691388:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573890.691389:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6680. 00000100:00000001:0.0:1788573890.691390:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573890.691391:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.691392:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573890.691395:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573890.691396:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573890.691397:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573890.691399:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573891.200910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573891.200920:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573891.200924:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573891.200932:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573891.200939:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573891.200944:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573891.200953:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573891.200956:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573891.200978:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573891.904992:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573891.904995:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573891.904998:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573892.224934:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573892.224939:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573892.224944:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573892.224950:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573892.224957:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573892.224964:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573892.224972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573892.224977:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573892.224983:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573893.248902:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573893.248907:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573893.248910:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573893.248914:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573893.248918:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573893.248922:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573893.248928:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573893.248931:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573893.248933:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573893.248987:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573893.248993:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573893.248994:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573893.248998:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573893.249001:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573893.249003:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573893.249005:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573893.249007:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573893.249009:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573894.272940:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573894.272948:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573894.272951:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573894.272957:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573894.272963:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573894.272968:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573894.272973:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573894.272996:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573894.273001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573894.273021:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573894.273026:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573894.273041:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573894.273057:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573894.273060:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573894.273062:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573894.273064:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573894.273066:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573894.273068:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573894.528923:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309254381, at 4309254400 00000100:00000010:0.0:1788573894.528939:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573894.977066:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573894.977076:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573894.977081:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573895.296894:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573895.296905:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573895.296911:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573895.296922:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573895.296930:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573895.296936:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.296945:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573895.296949:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.297104:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.809149:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573895.809153:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573895.809157:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573895.809166:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516629632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.809181:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573895.809184:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573895.809189:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573895.809192:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573895.809196:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573895.809203:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516629632:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573895.809218:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573895.809223:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573895.809226:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573895.809228:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573895.809231:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573895.809235:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573895.809243:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573895.809247:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573895.809255:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573895.809262:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516629632, portal 25 00000100:00000001:0.0:1788573895.809267:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573895.809269:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788573895.809276:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516629632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.809288:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573895.809291:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573895.809294:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573895.809298:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516629632, offset 0 00000400:00000200:0.0:1788573895.809306:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573895.809313:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573895.809319:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573895.809326:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573895.809331:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573895.809335:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573895.809338:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573895.809345:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546571:546571:256:4294967295] 192.168.202.128@tcp LPNI seq info [546494:546494:8:4294967295] 00000400:00000200:0.0:1788573895.809352:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573895.809363:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573895.809375:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573895.809381:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573895.809390:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573895.809402:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573895.809418:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573895.809423:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573895.809430:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573895.809437:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573895.809444:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573895.809448:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573895.809453:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573895.809461:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516629632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.809476:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573895.809481:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516629632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.809493:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.809498:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573895.809514:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573895.809523:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573895.809529:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309259501 00000100:00000010:0.0:1788573895.809538:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573895.809549:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573895.809557:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309259501 00000100:00000010:0.0:1788573895.809571:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50a80. 00000100:00000001:0.0:1788573895.809576:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573895.809581:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573895.809585:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.809587:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573895.809592:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573895/real 1788573895] req@ffff8fc4ac822d80 x1875440516629632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573895.809607:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573895.809611:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573895.809623:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573895.809628:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573895.809631:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573895.809635:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573895.809640:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516629632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573895.809656:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516629632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.809675:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573895.809678:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.809681:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573895.809685:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573895.809690:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516629632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.809698:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573895.809705:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516629632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.809720:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573895.809722:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.809723:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573895.809724:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.809725:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573895.809728:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573895.809735:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573895.809744:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573895.809747:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573895.809753:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516629632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573895.809759:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516629632:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573895.809774:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.809780:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573895.809785:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516629632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.809795:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573895.809798:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573895.809802:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573895.809838:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573895.809842:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573895.809845:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573895.809848:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573895.809852:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788573895.809858:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573895.809860:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573895.809864:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788573895.809871:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573895.809873:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.809879:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.809888:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573895.809892:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.809895:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573895.809897:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573895.809900:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573895.809906:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516629760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.809917:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573895.809919:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573895.809924:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573895.809927:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573895.809931:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573895.809934:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516629760:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573895.809937:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573895.809941:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573895.809943:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573895.809952:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573895.809958:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573895.809960:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573895.809968:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573895.809974:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573895.809979:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573895.809985:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516629760, portal 10 00000100:00000001:0.0:1788573895.809989:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573895.809994:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788573895.810000:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516629760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810007:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573895.810010:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573895.810026:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573895.810030:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516629760, offset 0 00000400:00000200:0.0:1788573895.810036:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573895.810044:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573895.810049:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573895.810054:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573895.810058:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573895.810063:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573895.810068:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573895.810074:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546572:546572:256:4294967295] 192.168.202.128@tcp LPNI seq info [546495:546495:8:4294967295] 00000400:00000200:0.0:1788573895.810081:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573895.810095:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573895.810101:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573895.810107:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573895.810114:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573895.810119:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573895.810138:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573895.810142:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573895.810150:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573895.810157:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573895.810162:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573895.810165:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573895.810170:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573895.810176:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516629760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810185:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573895.810189:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516629760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810198:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.810202:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573895.810207:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573895.810212:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573895.810218:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573895.810222:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309255699 00000100:00000001:0.0:1788573895.810226:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309259501 00000100:00000001:0.0:1788573895.810229:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309255699 00000100:00000001:0.0:1788573895.810234:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309259501 00000100:00000001:0.0:1788573895.810237:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573895.810239:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.810246:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573895.810252:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573895/real 1788573895] req@ffff8fc4ac821f80 x1875440516629760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573895.810265:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573895.810268:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573895.810270:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573895.810273:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573895.810277:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573895.810280:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573895.810292:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516629760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573895.810305:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516629760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810318:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573895.810320:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810321:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573895.810323:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573895.810328:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516629760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810337:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573895.810343:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516629760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810352:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573895.810358:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810359:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573895.810360:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810362:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573895.810364:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573895.810368:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573895.810371:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573895.810374:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573895.810382:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516629760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573895.810401:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516629760:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573895.810406:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573895.810408:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516629888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810436:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573895.810439:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573895.810442:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573895.810444:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573895.810449:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573895.810454:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516629888:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573895.810462:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573895.810464:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573895.810468:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573895.810470:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573895.810474:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573895.810476:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573895.810482:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573895.810485:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573895.810488:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573895.810490:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516629888, portal 10 00000100:00000001:0.0:1788573895.810492:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573895.810493:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788573895.810498:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516629888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810503:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573895.810506:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573895.810508:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573895.810510:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516629888, offset 0 00000400:00000200:0.0:1788573895.810515:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573895.810519:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573895.810523:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573895.810526:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573895.810528:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573895.810531:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573895.810533:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573895.810542:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546573:546573:256:4294967295] 192.168.202.128@tcp LPNI seq info [546496:546496:8:4294967295] 00000400:00000200:0.0:1788573895.810547:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573895.810554:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573895.810559:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573895.810563:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573895.810566:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573895.810568:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573895.810575:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573895.810576:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573895.810578:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573895.810580:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573895.810583:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573895.810584:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573895.810586:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573895.810589:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516629888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810594:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573895.810597:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516629888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810601:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.810602:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573895.810604:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573895.810607:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573895.810609:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573895.810610:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309255706 00000100:00000001:0.0:1788573895.810612:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309259501 00000100:00000001:0.0:1788573895.810614:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309255706 00000100:00000001:0.0:1788573895.810615:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309259501 00000100:00000001:0.0:1788573895.810616:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573895.810621:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.810623:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573895.810626:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573895/real 1788573895] req@ffff8fc4ac823480 x1875440516629888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573895.810632:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573895.810633:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573895.810634:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573895.810635:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573895.810636:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573895.810638:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573895.810640:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516629888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573895.810645:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516629888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810649:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573895.810650:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810651:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573895.810652:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573895.810654:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516629888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810658:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573895.810660:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516629888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810665:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573895.810666:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810667:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573895.810668:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810669:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573895.810672:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573895.810673:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573895.810678:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573895.810680:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573895.810683:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516629888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573895.810688:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516629888:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573895.810690:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810691:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573895.810693:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516629760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810698:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573895.810699:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573895.810700:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573895.810702:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573895.810703:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573895.810704:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573895.810705:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573895.810707:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788573895.810708:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573895.810709:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573895.810710:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788573895.810712:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573895.810713:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810714:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573895.810716:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516629888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810720:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573895.810721:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573895.810722:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573895.810723:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573895.810724:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573895.810724:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573895.810725:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573895.810729:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788573895.810731:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573895.810732:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573895.810732:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788573895.810734:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573895.810735:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810736:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810739:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573895.810740:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.810741:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573895.810742:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573895.810743:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573895.810745:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516630016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810750:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573895.810751:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573895.810752:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573895.810754:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573895.810755:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573895.810757:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516630016:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573895.810759:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573895.810760:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573895.810762:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573895.810763:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573895.810765:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573895.810766:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573895.810769:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573895.810771:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573895.810773:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573895.810775:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516630016, portal 4 00000100:00000001:0.0:1788573895.810776:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573895.810777:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788573895.810781:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516630016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810792:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573895.810794:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573895.810795:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573895.810797:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516630016, offset 0 00000400:00000200:0.0:1788573895.810800:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573895.810802:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573895.810805:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573895.810807:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573895.810824:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573895.810826:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573895.810827:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573895.810830:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546574:546574:256:4294967295] 192.168.202.128@tcp LPNI seq info [546497:546497:8:4294967295] 00000400:00000200:0.0:1788573895.810833:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573895.810837:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573895.810840:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573895.810843:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573895.810845:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573895.810847:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573895.810854:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573895.810856:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573895.810858:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573895.810860:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573895.810862:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573895.810863:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573895.810865:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573895.810868:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516630016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810873:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573895.810875:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516630016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810888:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.810890:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573895.810892:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573895.810894:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573895.810897:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573895.810898:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309255847 00000100:00000001:0.0:1788573895.810900:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309259501 00000100:00000001:0.0:1788573895.810902:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309255847 00000100:00000001:0.0:1788573895.810903:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309259501 00000100:00000001:0.0:1788573895.810904:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573895.810905:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.810907:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573895.810909:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573895/real 1788573895] req@ffff8fc4ac822300 x1875440516630016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573895.810914:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573895.810915:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573895.810916:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573895.810917:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573895.810918:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573895.810920:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573895.810922:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516630016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573895.810927:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516630016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810931:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573895.810932:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810933:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573895.810934:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573895.810936:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516630016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810944:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573895.810947:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516630016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810951:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573895.810952:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810953:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573895.810954:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.810955:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573895.810956:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573895.810958:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573895.810960:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573895.810961:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573895.810964:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516630016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573895.810969:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516630016:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573895.810971:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573895.810973:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516630144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.810977:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573895.810978:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573895.810979:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573895.810980:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573895.810982:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573895.810983:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516630144:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573895.810985:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573895.810986:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573895.810990:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573895.810991:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573895.810993:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573895.810994:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573895.810997:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573895.810998:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573895.811000:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573895.811002:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516630144, portal 4 00000100:00000001:0.0:1788573895.811003:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573895.811004:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788573895.811008:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516630144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.811028:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573895.811030:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573895.811032:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573895.811033:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516630144, offset 0 00000400:00000200:0.0:1788573895.811036:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573895.811039:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573895.811042:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573895.811044:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573895.811046:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573895.811047:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573895.811049:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573895.811051:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546575:546575:256:4294967295] 192.168.202.128@tcp LPNI seq info [546498:546498:8:4294967295] 00000400:00000200:0.0:1788573895.811054:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573895.811058:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573895.811061:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573895.811063:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573895.811065:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573895.811067:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573895.811075:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573895.811080:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56900. 00000400:00000200:0.0:1788573895.811082:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573895.811084:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573895.811087:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9f68 00000400:00000010:0.0:1788573895.811088:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9f68. 00000100:00000001:0.0:1788573895.811090:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573895.811093:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516630144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.811098:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573895.811100:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516630144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.811104:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.811105:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573895.811107:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573895.811109:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573895.811111:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573895.811112:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309255849 00000100:00000001:0.0:1788573895.811114:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309259501 00000100:00000001:0.0:1788573895.811116:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309255849 00000100:00000001:0.0:1788573895.811117:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309259501 00000100:00000001:0.0:1788573895.811119:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573895.811120:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.811121:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573895.811123:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573895/real 1788573895] req@ffff8fc4ac820700 x1875440516630144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573895.811128:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573895.811129:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573895.811130:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573895.811132:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573895.811133:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573895.811137:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573895.811140:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516630144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573895.811145:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516630144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.811149:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573895.811150:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.811151:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573895.811152:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573895.811154:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516630144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.811158:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573895.811161:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516630144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.811165:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573895.811167:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.811168:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573895.811168:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.811170:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573895.811171:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573895.811172:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573895.811174:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573895.811175:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573895.811178:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516630144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573895.811182:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516630144:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573895.811184:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.811186:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573895.811190:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516630016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.811195:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573895.811196:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573895.811198:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573895.811199:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573895.811200:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573895.811201:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573895.811202:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573895.811204:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788573895.811205:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573895.811206:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573895.811207:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788573895.811209:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573895.811209:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.811210:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573895.811212:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516630144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573895.811217:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573895.811217:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573895.811218:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573895.811220:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573895.811220:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573895.811221:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573895.811222:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573895.811223:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788573895.811224:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573895.811225:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573895.811225:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788573895.811227:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573895.811228:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.811229:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573895.811231:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573895.811232:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573895.811233:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573895.811234:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573896.320960:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573896.320967:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573896.320971:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573896.320976:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573896.320982:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573896.320986:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573896.320990:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573896.320992:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573896.320997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573897.025064:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573897.025067:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573897.025069:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573897.344929:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573897.344937:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573897.344941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573897.344948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573897.344953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573897.344961:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573897.344968:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573897.344971:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573897.344975:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573898.368923:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573898.368927:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573898.368931:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573898.368936:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573898.368941:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573898.368945:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573898.368993:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573898.368997:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573898.369001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573899.648923:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309259501, at 4309259520 00000100:00000010:0.0:1788573899.648963:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573900.096974:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573900.096979:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573900.096983:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573900.417024:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573900.417035:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573900.417051:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573900.417057:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573900.417064:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573900.417068:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573900.417072:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573900.417075:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573900.417078:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573900.929255:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573900.929262:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573900.929267:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573900.929277:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516630272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573900.929298:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573900.929302:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573900.929305:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573900.929308:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573900.929311:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573900.929317:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516630272:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573900.929325:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573900.929331:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573900.929336:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573900.929342:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573900.929351:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573900.929356:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573900.929365:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573900.929371:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573900.929382:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573900.929390:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516630272, portal 25 00000100:00000001:0.0:1788573900.929397:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573900.929401:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788573900.929413:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516630272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573900.929426:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573900.929432:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573900.929443:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9f68. 00000100:00000200:0.0:1788573900.929449:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516630272, offset 0 00000400:00000200:0.0:1788573900.929463:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573900.929475:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573900.929486:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573900.929503:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573900.929509:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573900.929516:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573900.929522:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573900.929528:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546576:546576:256:4294967295] 192.168.202.128@tcp LPNI seq info [546499:546499:8:4294967295] 00000400:00000200:0.0:1788573900.929536:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573900.929561:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573900.929577:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573900.929586:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56900. 00000800:00000200:0.0:1788573900.929598:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573900.929627:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573900.929643:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573900.929648:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309264621 00000100:00000010:0.0:1788573900.929654:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573900.929663:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573900.929671:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309264621 00000100:00000010:0.0:1788573900.929690:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50720. 00000100:00000001:0.0:1788573900.929698:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573900.929704:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573900.929709:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573900.929716:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573900.929723:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573900.929732:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573900.929736:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573900.929749:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573900.929753:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573900.929757:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573900.929773:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516630400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573900.929800:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573900.929804:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573900.929837:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573900.929839:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573900.929844:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573900.929850:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516630400:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573900.929858:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573900.929862:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573900.929866:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573900.929871:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573900.929877:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573900.929884:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573900.929893:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573900.929898:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915948. 00000400:00000010:0.0:1788573900.929905:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573900.929927:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516630400, portal 10 00000100:00000001:0.0:1788573900.929933:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573900.929937:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788573900.929950:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516630400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573900.929982:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573900.929993:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573900.929999:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed97f8. 00000100:00000200:0.0:1788573900.930005:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516630400, offset 0 00000400:00000200:0.0:1788573900.930016:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573900.930025:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573900.930034:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573900.930039:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573900.930046:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573900.930052:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573900.930066:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573900.930071:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546577:546577:255:4294967295] 192.168.202.128@tcp LPNI seq info [546500:546500:7:4294967295] 00000400:00000200:0.0:1788573900.930080:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573900.930096:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573900.930111:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573900.930118:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56300. 00000800:00000200:0.0:1788573900.930130:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573900.930149:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573900.930156:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573900.930165:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573900.930171:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309260819 00000100:00000001:0.0:1788573900.930177:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309264621 00000100:00000001:0.0:1788573900.930184:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309260819 00000100:00000001:0.0:1788573900.930190:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309264621 00000100:00000001:0.0:1788573900.930195:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573900.930201:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573900.930205:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573900.930207:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573900.930213:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573900.930217:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573900.930223:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573900.930228:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573900.930233:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573900.930246:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516630528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573900.930274:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573900.930276:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573900.930282:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573900.930287:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573900.930294:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573900.930307:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516630528:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573900.930322:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573900.930326:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573900.930331:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573900.930336:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573900.930342:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573900.930349:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573900.930359:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573900.930366:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573900.930373:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573900.930378:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516630528, portal 10 00000100:00000001:0.0:1788573900.930384:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573900.930386:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788573900.930399:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516630528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573900.930417:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573900.930425:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573900.930431:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9770. 00000100:00000200:0.0:1788573900.930436:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516630528, offset 0 00000400:00000200:0.0:1788573900.930448:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573900.930457:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573900.930463:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573900.930467:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573900.930471:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573900.930475:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573900.930478:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573900.930483:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546578:546578:254:4294967295] 192.168.202.128@tcp LPNI seq info [546501:546501:6:4294967295] 00000400:00000200:0.0:1788573900.930489:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573900.930501:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573900.930507:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573900.930512:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788573900.930518:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573900.930532:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573900.930536:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573900.930540:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573900.930543:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309260826 00000100:00000001:0.0:1788573900.930546:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309264621 00000100:00000001:0.0:1788573900.930549:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309260826 00000100:00000001:0.0:1788573900.930551:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309264621 00000100:00000001:0.0:1788573900.930553:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573900.930556:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573900.930558:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573900.930563:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516630656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573900.930571:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573900.930573:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573900.930575:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573900.930577:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573900.930579:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573900.930582:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516630656:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573900.930586:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573900.930588:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573900.930590:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573900.930592:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573900.930595:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573900.930597:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573900.930601:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573900.930604:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573900.930608:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573900.930611:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516630656, portal 4 00000100:00000001:0.0:1788573900.930613:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573900.930614:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788573900.930620:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516630656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573900.930632:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573900.930634:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573900.930637:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed94c8. 00000100:00000200:0.0:1788573900.930640:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516630656, offset 0 00000400:00000200:0.0:1788573900.930644:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573900.930648:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573900.930652:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573900.930655:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573900.930658:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573900.930660:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573900.930662:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573900.930666:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546579:546579:253:4294967295] 192.168.202.128@tcp LPNI seq info [546502:546502:5:4294967295] 00000400:00000200:0.0:1788573900.930670:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573900.930676:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573900.930681:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573900.930686:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56b00. 00000800:00000200:0.0:1788573900.930690:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573900.930696:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573900.930700:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573900.930703:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573900.930705:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309260966 00000100:00000001:0.0:1788573900.930708:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309264621 00000100:00000001:0.0:1788573900.930710:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309260966 00000100:00000001:0.0:1788573900.930712:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309264621 00000100:00000001:0.0:1788573900.930715:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573900.930717:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573900.930720:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573900.930722:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573900.930726:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573900.930735:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573900.930737:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573900.930739:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573900.930740:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573900.930745:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516630784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573900.930753:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573900.930755:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573900.930757:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573900.930759:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573900.930762:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573900.930765:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516630784:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573900.930768:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573900.930770:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573900.930772:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573900.930774:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573900.930778:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573900.930779:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573900.930782:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573900.930784:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915790. 00000400:00000010:0.0:1788573900.930787:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9a18. 00000100:00000200:0.0:1788573900.930790:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516630784, portal 4 00000100:00000001:0.0:1788573900.930792:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573900.930794:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788573900.930800:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516630784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573900.930807:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573900.930833:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573900.930836:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9110. 00000100:00000200:0.0:1788573900.930839:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516630784, offset 0 00000400:00000200:0.0:1788573900.930844:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573900.930848:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573900.930852:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573900.930861:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573900.930864:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573900.930866:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573900.930869:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573900.930872:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546580:546580:252:4294967295] 192.168.202.128@tcp LPNI seq info [546503:546503:4:4294967295] 00000400:00000200:0.0:1788573900.930876:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573900.930882:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573900.930886:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573900.930890:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56c00. 00000800:00000200:0.0:1788573900.930893:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573900.930899:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573900.930902:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573900.930905:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573900.930921:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309260969 00000100:00000001:0.0:1788573900.930925:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309264621 00000100:00000001:0.0:1788573900.930927:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309260969 00000100:00000001:0.0:1788573900.930929:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309264621 00000100:00000001:0.0:1788573900.930931:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573900.930934:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573900.930936:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573900.930938:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573900.930943:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573900.930945:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573900.930947:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573900.930948:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573900.930950:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573900.930952:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573901.440896:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573901.440916:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573901.440917:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573901.440924:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573901.440929:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573901.440952:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573901.440962:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573901.440975:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573901.440980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573902.145039:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573902.145042:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573902.145044:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573902.464923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573902.464928:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573902.464931:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573902.464935:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573902.464942:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573902.464946:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573902.464951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573902.464956:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573902.464960:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573902.464993:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573902.464997:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573902.464999:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573902.465001:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573902.465003:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573902.465006:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573902.465008:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573902.465010:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573902.465012:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573903.488904:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573903.488913:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573903.488917:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573903.488922:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573903.488928:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573903.488932:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573903.488962:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573903.488966:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573903.488970:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.001429:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573904.001439:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573904.001449:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573904.001472:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573900/real 1788573904] req@ffff8fc4901a4e00 x1875440516630400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573904.001524:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573904.001529:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573904.001534:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915948. 00000400:00000200:0.0:1788573904.001550:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573904.001560:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573904.001592:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573904.001607:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516630400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573904.001643:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516630400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.001694:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573904.001698:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.001701:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573904.001704:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573904.001712:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516630400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.001759:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573904.001774:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516630400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.001785:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573904.001789:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.001791:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573904.001793:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.001797:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573904.001801:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573904.001806:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573904.001909:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573904.001920:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573904.001951:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516630400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573904.001985:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516630400:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573904.002005:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573904.002017:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573900/real 1788573904] req@ffff8fc4901a6300 x1875440516630272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573904.002059:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573904.002064:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573904.002071:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573904.002078:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9cc0 00000400:00000010:0.0:1788573904.002081:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9cc0. 00000100:00000001:0.0:1788573904.002086:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573904.002093:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516630272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573904.002105:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516630272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002119:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573904.002121:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002124:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573904.002125:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573904.002130:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516630272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002138:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573904.002142:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516630272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002151:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573904.002153:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002156:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573904.002163:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002165:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573904.002167:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573904.002171:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573904.002176:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573904.002179:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573904.002183:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516630272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573904.002193:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516630272:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573904.002198:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.002202:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573904.002207:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516630400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002217:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573904.002222:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573904.002228:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573904.002234:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573904.002237:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573904.002241:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573904.002244:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573904.002248:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788573904.002255:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573904.002257:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573904.002259:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788573904.002269:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573904.002271:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002273:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573904.002277:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516630272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002285:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573904.002286:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573904.002288:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573904.002298:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573904.002300:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573904.002301:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573904.002304:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573904.002306:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788573904.002309:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573904.002310:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573904.002312:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788573904.002317:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573904.002319:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002322:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.002325:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573904.002326:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573904.002328:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573904.002334:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573900/real 1788573904] req@ffff8fc4901a7480 x1875440516630784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573904.002347:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573904.002350:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573904.002353:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915790. 00000400:00000200:0.0:1788573904.002356:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9a18 00000400:00000010:0.0:1788573904.002358:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9a18. 00000100:00000001:0.0:1788573904.002363:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573904.002368:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516630784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573904.002378:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516630784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002385:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573904.002386:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002389:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573904.002390:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573904.002394:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516630784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002402:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573904.002414:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516630784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002422:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573904.002425:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002427:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573904.002428:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002430:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573904.002433:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573904.002435:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573904.002440:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573904.002442:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573904.002447:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516630784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573904.002454:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516630784:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573904.002458:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573904.002461:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573900/real 1788573904] req@ffff8fc4901a4700 x1875440516630528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573904.002467:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573904.002469:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573904.002471:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573904.002473:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573904.002474:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573904.002476:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573904.002479:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516630528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573904.002488:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516630528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002495:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573904.002503:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002505:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573904.002507:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573904.002511:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516630528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002519:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573904.002523:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516630528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002531:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573904.002534:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002535:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573904.002537:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002539:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573904.002542:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573904.002545:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573904.002549:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573904.002552:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573904.002556:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516630528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573904.002565:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516630528:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573904.002570:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573904.002575:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573900/real 1788573904] req@ffff8fc4901a5880 x1875440516630656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573904.002584:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573904.002586:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573904.002588:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573904.002591:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573904.002593:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573904.002602:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573904.002606:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516630656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573904.002615:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516630656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002622:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573904.002624:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002626:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573904.002628:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573904.002632:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516630656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002639:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573904.002644:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516630656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002651:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573904.002654:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002655:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573904.002656:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002658:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573904.002661:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573904.002664:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573904.002667:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573904.002670:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573904.002674:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516630656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573904.002683:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516630656:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573904.002688:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002690:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573904.002699:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516630784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002707:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573904.002708:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573904.002710:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573904.002715:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573904.002716:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573904.002718:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573904.002720:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573904.002722:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5c00. 02000000:00000001:0.0:1788573904.002725:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573904.002726:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573904.002728:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788573904.002732:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573904.002733:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002735:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573904.002739:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516630528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002746:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573904.002747:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573904.002748:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573904.002751:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573904.002752:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573904.002753:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573904.002756:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573904.002757:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788573904.002760:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573904.002761:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573904.002762:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788573904.002765:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573904.002766:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002768:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573904.002772:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516630656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573904.002779:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573904.002786:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573904.002788:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573904.002791:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573904.002792:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573904.002794:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573904.002796:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573904.002797:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788573904.002800:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573904.002801:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573904.002802:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788573904.002806:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573904.002836:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002841:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.002862:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573904.002865:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.002868:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573904.002870:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.512924:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573904.512931:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573904.512936:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573904.512942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573904.512948:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573904.512955:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.512960:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573904.512964:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.512981:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573904.513026:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573904.513029:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573904.513038:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573904.513048:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573904.513054:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573904.513059:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.513062:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573904.513064:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573904.513067:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573904.768933:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309264621, at 4309264640 00000100:00000010:0.0:1788573904.768957:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573905.217041:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573905.217059:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573905.217062:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573905.536930:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573905.536940:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573905.536945:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573905.536950:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573905.536955:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573905.536962:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573905.536993:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573905.536998:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573905.537003:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573905.537026:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573905.537035:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573905.537037:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573905.537039:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573905.537042:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573905.537044:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573905.537046:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573905.537048:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573905.537050:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.049251:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573906.049258:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573906.049263:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573906.049276:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516630912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.049288:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573906.049293:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573906.049301:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573906.049307:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573906.049313:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573906.049323:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516630912:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573906.049346:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573906.049352:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573906.049355:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573906.049361:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573906.049369:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573906.049382:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573906.049393:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573906.049401:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573906.049412:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573906.049422:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516630912, portal 25 00000100:00000001:0.0:1788573906.049428:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573906.049432:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788573906.049445:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516630912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.049462:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573906.049470:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573906.049476:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573906.049482:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516630912, offset 0 00000400:00000200:0.0:1788573906.049494:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573906.049504:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573906.049515:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573906.049524:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573906.049533:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573906.049538:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573906.049541:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573906.049546:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546581:546581:256:4294967295] 192.168.202.128@tcp LPNI seq info [546504:546504:8:4294967295] 00000400:00000200:0.0:1788573906.049551:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573906.049561:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573906.049572:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573906.049581:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573906.049595:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573906.049604:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573906.049635:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573906.049639:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573906.049646:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573906.049658:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573906.049677:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573906.049683:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573906.049691:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573906.049701:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516630912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.049713:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573906.049717:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516630912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.049734:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.049738:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573906.049745:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573906.049753:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573906.049760:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309269741 00000100:00000010:0.0:1788573906.049768:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573906.049779:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573906.049788:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309269741 00000100:00000010:0.0:1788573906.049795:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50f00. 00000100:00000001:0.0:1788573906.049801:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573906.049853:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573906.049857:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.049860:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573906.049868:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573906/real 1788573906] req@ffff8fc4ac9f6a00 x1875440516630912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573906.049908:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573906.049912:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573906.049922:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573906.049929:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573906.049934:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573906.049940:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573906.049949:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516630912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573906.049969:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516630912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.049994:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573906.049997:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.050000:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573906.050005:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573906.050017:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516630912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.050031:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573906.050042:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516630912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.050062:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573906.050070:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.050075:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573906.050079:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.050085:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573906.050089:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573906.050096:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573906.050106:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573906.050112:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573906.050122:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516630912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573906.050145:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516630912:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573906.050154:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.050157:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573906.050162:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516630912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.050173:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573906.050178:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573906.050189:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573906.050194:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573906.050198:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573906.050202:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573906.050215:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573906.050221:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34400. 02000000:00000001:0.0:1788573906.050228:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573906.050232:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573906.050236:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788573906.050242:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573906.050248:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.050254:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.050265:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573906.050270:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.050276:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573906.050281:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573906.050285:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573906.050295:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4e00 x1875440516631040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.050315:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573906.050320:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573906.050324:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573906.050329:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573906.050334:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573906.050343:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631040:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573906.050347:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573906.050351:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573906.050356:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573906.050361:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573906.050368:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573906.050373:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573906.050382:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573906.050388:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573906.050396:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573906.050404:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516631040, portal 10 00000100:00000001:0.0:1788573906.050418:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573906.050422:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605324288 : -123400104227328 : ffff8fc4ac9f4e00) 00000100:00000040:0.0:1788573906.050434:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4e00 x1875440516631040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.050449:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573906.050457:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573906.050465:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573906.050473:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516631040, offset 0 00000400:00000200:0.0:1788573906.050484:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573906.050494:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573906.050502:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573906.050508:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573906.050515:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573906.050523:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573906.050531:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573906.050540:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546582:546582:256:4294967295] 192.168.202.128@tcp LPNI seq info [546505:546505:8:4294967295] 00000400:00000200:0.0:1788573906.050549:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573906.050564:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573906.050577:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573906.050584:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573906.050594:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573906.050599:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573906.050614:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573906.050616:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573906.050620:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573906.050624:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573906.050629:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573906.050631:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573906.050633:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573906.050637:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4e00 x1875440516631040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.050658:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573906.050662:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4e00 x1875440516631040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.050668:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.050669:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573906.050674:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573906.050677:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573906.050681:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573906.050684:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309265939 00000100:00000001:0.0:1788573906.050687:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309269741 00000100:00000001:0.0:1788573906.050689:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309265939 00000100:00000001:0.0:1788573906.050692:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309269741 00000100:00000001:0.0:1788573906.050694:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573906.050696:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.050699:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573906.050702:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573906/real 1788573906] req@ffff8fc4ac9f4e00 x1875440516631040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573906.050708:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573906.050709:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573906.050711:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573906.050713:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573906.050714:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573906.050716:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573906.050718:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4e00 x1875440516631040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573906.050724:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4e00 x1875440516631040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.050728:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573906.050733:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.050735:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573906.050736:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573906.050738:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4e00 x1875440516631040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.050743:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573906.050746:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4e00 x1875440516631040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.050750:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573906.050752:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.050754:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573906.050755:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.050757:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573906.050759:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573906.050762:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573906.050765:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573906.050768:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573906.050772:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4e00 x1875440516631040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573906.050781:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631040:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573906.050786:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573906.050789:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5180 x1875440516631168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.050796:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573906.050798:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573906.050800:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573906.050802:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573906.050840:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573906.050850:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631168:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573906.050854:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573906.050856:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573906.050858:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573906.050860:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573906.050864:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573906.050866:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573906.050870:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573906.050873:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573906.050877:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573906.050880:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516631168, portal 10 00000100:00000001:0.0:1788573906.050882:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573906.050883:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605325184 : -123400104226432 : ffff8fc4ac9f5180) 00000100:00000040:0.0:1788573906.050896:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5180 x1875440516631168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.050903:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573906.050906:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573906.050909:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573906.050911:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516631168, offset 0 00000400:00000200:0.0:1788573906.050916:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573906.050920:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573906.050924:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573906.050927:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573906.050930:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573906.050933:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573906.050935:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573906.050939:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546583:546583:256:4294967295] 192.168.202.128@tcp LPNI seq info [546506:546506:8:4294967295] 00000400:00000200:0.0:1788573906.050943:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573906.050950:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573906.050955:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573906.050958:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573906.050970:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573906.050973:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573906.050985:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573906.050987:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573906.050990:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573906.050994:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573906.050998:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573906.051000:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573906.051003:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573906.051007:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5180 x1875440516631168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051016:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573906.051020:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5180 x1875440516631168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051027:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.051028:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573906.051032:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573906.051035:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573906.051039:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573906.051041:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309265947 00000100:00000001:0.0:1788573906.051044:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309269741 00000100:00000001:0.0:1788573906.051046:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309265947 00000100:00000001:0.0:1788573906.051049:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309269741 00000100:00000001:0.0:1788573906.051051:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573906.051053:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.051055:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573906.051058:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573906/real 1788573906] req@ffff8fc4ac9f5180 x1875440516631168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573906.051064:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573906.051069:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573906.051071:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573906.051073:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573906.051074:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573906.051076:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573906.051078:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516631168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573906.051084:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516631168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051088:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573906.051089:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051090:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573906.051091:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573906.051094:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516631168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051098:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573906.051100:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516631168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051105:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573906.051107:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051108:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573906.051109:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051110:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573906.051112:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573906.051114:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573906.051115:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573906.051117:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573906.051119:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516631168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573906.051135:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631168:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573906.051139:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573906.051143:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7100 x1875440516631296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051150:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573906.051152:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573906.051155:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573906.051156:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573906.051158:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573906.051162:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631296:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573906.051165:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573906.051168:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573906.051169:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573906.051171:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573906.051175:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573906.051177:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573906.051181:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573906.051184:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573906.051188:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573906.051191:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516631296, portal 4 00000100:00000001:0.0:1788573906.051193:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573906.051195:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605333248 : -123400104218368 : ffff8fc4ac9f7100) 00000100:00000040:0.0:1788573906.051207:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7100 x1875440516631296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051214:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573906.051216:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573906.051219:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573906.051222:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516631296, offset 0 00000400:00000200:0.0:1788573906.051226:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573906.051237:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573906.051241:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573906.051244:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573906.051247:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573906.051249:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573906.051252:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573906.051255:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546584:546584:256:4294967295] 192.168.202.128@tcp LPNI seq info [546507:546507:8:4294967295] 00000400:00000200:0.0:1788573906.051260:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573906.051267:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573906.051272:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573906.051276:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573906.051280:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573906.051283:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573906.051300:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573906.051302:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573906.051305:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573906.051309:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573906.051313:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573906.051315:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573906.051318:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573906.051323:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7100 x1875440516631296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051331:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573906.051334:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7100 x1875440516631296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051341:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.051343:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573906.051347:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573906.051350:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573906.051353:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573906.051362:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309266087 00000100:00000001:0.0:1788573906.051365:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309269741 00000100:00000001:0.0:1788573906.051367:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309266087 00000100:00000001:0.0:1788573906.051369:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309269741 00000100:00000001:0.0:1788573906.051371:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573906.051374:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.051376:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573906.051381:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573906/real 1788573906] req@ffff8fc4ac9f7100 x1875440516631296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573906.051390:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573906.051392:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573906.051394:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573906.051396:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573906.051398:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573906.051401:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573906.051404:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7100 x1875440516631296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573906.051413:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7100 x1875440516631296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051421:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573906.051422:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051424:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573906.051426:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573906.051430:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7100 x1875440516631296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051437:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573906.051442:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7100 x1875440516631296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051449:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573906.051457:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051459:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573906.051460:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051462:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573906.051464:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573906.051468:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573906.051472:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573906.051474:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573906.051504:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7100 x1875440516631296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573906.051513:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631296:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573906.051517:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051519:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573906.051522:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4e00 x1875440516631040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051530:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573906.051531:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573906.051534:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573906.051537:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573906.051539:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573906.051540:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573906.051542:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573906.051544:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36c00. 02000000:00000001:0.0:1788573906.051547:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573906.051549:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573906.051550:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4e00. 00000100:00000001:0.0:1788573906.051553:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573906.051555:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051557:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573906.051560:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516631168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051577:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573906.051578:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573906.051580:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573906.051581:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573906.051582:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573906.051583:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573906.051584:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573906.051585:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788573906.051587:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573906.051587:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573906.051588:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788573906.051590:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573906.051591:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051592:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573906.051595:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7100 x1875440516631296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051602:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573906.051604:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573906.051605:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573906.051607:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573906.051608:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573906.051609:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573906.051611:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573906.051613:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788573906.051615:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573906.051616:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573906.051617:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7100. 00000100:00000001:0.0:1788573906.051620:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573906.051621:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051624:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051628:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573906.051630:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.051632:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573906.051634:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573906.051635:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573906.051645:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4000 x1875440516631424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051651:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573906.051652:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573906.051654:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573906.051655:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573906.051657:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573906.051659:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631424:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573906.051661:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573906.051663:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573906.051664:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573906.051665:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573906.051667:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573906.051669:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573906.051673:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573906.051675:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573906.051679:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573906.051682:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516631424, portal 4 00000100:00000001:0.0:1788573906.051684:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573906.051686:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605320704 : -123400104230912 : ffff8fc4ac9f4000) 00000100:00000040:0.0:1788573906.051691:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4000 x1875440516631424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051696:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573906.051697:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573906.051699:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573906.051701:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516631424, offset 0 00000400:00000200:0.0:1788573906.051703:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573906.051706:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573906.051709:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573906.051711:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573906.051713:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573906.051715:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573906.051716:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573906.051723:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546585:546585:256:4294967295] 192.168.202.128@tcp LPNI seq info [546508:546508:8:4294967295] 00000400:00000200:0.0:1788573906.051726:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573906.051740:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573906.051743:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573906.051746:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573906.051749:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573906.051751:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788573906.051770:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573906.051772:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573906.051774:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573906.051777:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573906.051779:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573906.051780:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573906.051782:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573906.051785:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4000 x1875440516631424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051791:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573906.051793:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4000 x1875440516631424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051797:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.051798:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573906.051800:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573906.051802:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573906.051837:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573906.051838:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309266090 00000100:00000001:0.0:1788573906.051840:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309269741 00000100:00000001:0.0:1788573906.051842:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309266090 00000100:00000001:0.0:1788573906.051844:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309269741 00000100:00000001:0.0:1788573906.051845:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573906.051851:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.051852:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573906.051856:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573906/real 1788573906] req@ffff8fc4ac9f4000 x1875440516631424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573906.051862:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573906.051863:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573906.051865:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573906.051866:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573906.051868:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573906.051869:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573906.051871:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4000 x1875440516631424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573906.051877:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4000 x1875440516631424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051881:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573906.051882:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051883:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573906.051884:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573906.051886:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4000 x1875440516631424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051890:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573906.051893:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4000 x1875440516631424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051897:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573906.051899:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051901:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573906.051902:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051904:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573906.051906:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573906.051908:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573906.051916:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573906.051919:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573906.051923:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4000 x1875440516631424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573906.051932:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631424:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573906.051936:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051938:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573906.051942:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4000 x1875440516631424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573931 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573906.051949:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573906.051951:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573906.051953:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573906.051956:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573906.051957:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573906.051959:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573906.051960:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573906.051961:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788573906.051963:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573906.051964:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573906.051965:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4000. 00000100:00000001:0.0:1788573906.051967:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573906.051968:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051969:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.051972:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573906.051973:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.051974:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573906.051976:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573906.560941:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573906.560946:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573906.560950:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573906.560954:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573906.560958:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573906.560963:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.560982:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573906.560984:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573906.560988:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573907.265028:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573907.265032:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573907.265035:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573907.584914:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573907.584923:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573907.584927:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573907.584935:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573907.584943:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573907.584950:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573907.584962:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573907.584967:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573907.584975:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573908.608999:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573908.609013:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573908.609023:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573908.609045:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573908.609052:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573908.609059:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573908.609065:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573908.609071:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573908.609079:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.632919:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573909.632924:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573909.632928:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573909.632936:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573909.632943:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573909.632948:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573909.632953:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573909.632956:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573909.632970:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573909.633004:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573909.633007:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573909.633008:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573909.633010:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573909.633021:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573909.633048:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573909.633794:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573909.633797:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573909.633829:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573909.888919:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309269741, at 4309269760 00000100:00000010:0.0:1788573909.888935:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573910.337006:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573910.337009:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573910.337012:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573911.169333:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573911.169342:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573911.169346:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573911.169364:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5500 x1875440516631552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.169390:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573911.169394:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573911.169405:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573911.169411:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573911.169422:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573911.169436:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516631552:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573911.169443:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573911.169447:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573911.169454:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573911.169461:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573911.169473:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573911.169481:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573911.169492:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573911.169500:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573911.169511:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573911.169525:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516631552, portal 25 00000100:00000001:0.0:1788573911.169531:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573911.169536:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605326080 : -123400104225536 : ffff8fc4ac9f5500) 00000100:00000040:0.0:1788573911.169556:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5500 x1875440516631552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.169616:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573911.169625:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573911.169638:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573911.169654:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516631552, offset 0 00000400:00000200:0.0:1788573911.169671:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573911.169684:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573911.169694:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573911.169700:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573911.169707:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573911.169712:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573911.169718:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573911.169734:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546586:546586:256:4294967295] 192.168.202.128@tcp LPNI seq info [546509:546509:8:4294967295] 00000400:00000200:0.0:1788573911.169753:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573911.169790:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573911.169849:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573911.169854:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573911.169861:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573911.169867:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573911.169893:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573911.169898:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573911.169910:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573911.169922:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573911.169940:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573911.169945:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573911.169956:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573911.169969:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5500 x1875440516631552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170006:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573911.170016:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5500 x1875440516631552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170071:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.170079:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573911.170092:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573911.170102:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573911.170109:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309274862 00000100:00000010:0.0:1788573911.170116:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573911.170134:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573911.170148:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309274862 00000100:00000010:0.0:1788573911.170153:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b505a0. 00000100:00000001:0.0:1788573911.170159:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573911.170169:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573911.170172:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.170183:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573911.170194:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573911/real 1788573911] req@ffff8fc4ac9f5500 x1875440516631552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573911.170213:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573911.170214:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573911.170221:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573911.170232:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573911.170239:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573911.170251:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573911.170263:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5500 x1875440516631552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573911.170303:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5500 x1875440516631552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170331:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573911.170336:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.170344:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573911.170350:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573911.170359:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5500 x1875440516631552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170386:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573911.170397:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5500 x1875440516631552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170431:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573911.170440:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.170452:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573911.170456:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.170458:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573911.170462:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573911.170470:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573911.170487:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573911.170496:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573911.170509:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5500 x1875440516631552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573911.170535:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516631552:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573911.170549:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.170553:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573911.170560:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5500 x1875440516631552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170574:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573911.170580:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573911.170590:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573911.170596:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573911.170602:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573911.170607:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573911.170613:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573911.170624:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788573911.170640:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573911.170652:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573911.170658:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5500. 00000100:00000001:0.0:1788573911.170666:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573911.170669:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.170682:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.170690:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573911.170693:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.170695:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573911.170697:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573911.170699:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573911.170704:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6d80 x1875440516631680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170710:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573911.170711:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573911.170713:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573911.170715:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573911.170716:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573911.170718:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631680:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573911.170720:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573911.170722:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573911.170723:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573911.170725:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573911.170727:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573911.170728:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573911.170732:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573911.170734:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573911.170737:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573911.170739:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516631680, portal 10 00000100:00000001:0.0:1788573911.170741:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573911.170742:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605332352 : -123400104219264 : ffff8fc4ac9f6d80) 00000100:00000040:0.0:1788573911.170746:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6d80 x1875440516631680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170751:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573911.170752:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573911.170758:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573911.170759:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516631680, offset 0 00000400:00000200:0.0:1788573911.170763:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573911.170781:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573911.170784:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573911.170787:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573911.170789:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573911.170791:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573911.170792:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573911.170795:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546587:546587:256:4294967295] 192.168.202.128@tcp LPNI seq info [546510:546510:8:4294967295] 00000400:00000200:0.0:1788573911.170798:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573911.170803:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573911.170806:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573911.170827:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573911.170830:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573911.170831:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573911.170840:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573911.170842:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573911.170844:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573911.170846:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573911.170849:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573911.170850:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573911.170852:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573911.170855:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6d80 x1875440516631680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170862:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573911.170866:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6d80 x1875440516631680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170873:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.170874:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573911.170884:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573911.170889:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573911.170893:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573911.170895:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309271060 00000100:00000001:0.0:1788573911.170898:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309274862 00000100:00000001:0.0:1788573911.170900:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309271060 00000100:00000001:0.0:1788573911.170903:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309274862 00000100:00000001:0.0:1788573911.170905:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573911.170907:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.170909:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573911.170913:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573911/real 1788573911] req@ffff8fc4ac9f6d80 x1875440516631680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573911.170920:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573911.170921:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573911.170922:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573911.170924:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573911.170925:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573911.170927:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573911.170929:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6d80 x1875440516631680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573911.170935:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6d80 x1875440516631680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170939:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573911.170940:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.170941:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573911.170942:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573911.170944:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6d80 x1875440516631680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170952:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573911.170956:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6d80 x1875440516631680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.170970:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573911.170973:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.170975:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573911.170977:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.170979:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573911.170981:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573911.170984:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573911.170988:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573911.170991:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573911.170995:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6d80 x1875440516631680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573911.171003:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631680:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573911.171006:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573911.171009:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5c00 x1875440516631808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171013:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573911.171014:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573911.171016:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573911.171017:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573911.171019:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573911.171020:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631808:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573911.171023:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573911.171025:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573911.171026:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573911.171027:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573911.171030:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573911.171031:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573911.171039:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573911.171041:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573911.171043:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573911.171046:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516631808, portal 10 00000100:00000001:0.0:1788573911.171047:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573911.171048:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605327872 : -123400104223744 : ffff8fc4ac9f5c00) 00000100:00000040:0.0:1788573911.171052:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5c00 x1875440516631808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171057:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573911.171058:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573911.171059:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573911.171061:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516631808, offset 0 00000400:00000200:0.0:1788573911.171064:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573911.171066:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573911.171069:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573911.171071:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573911.171072:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573911.171074:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573911.171076:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573911.171078:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546588:546588:256:4294967295] 192.168.202.128@tcp LPNI seq info [546511:546511:8:4294967295] 00000400:00000200:0.0:1788573911.171081:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573911.171085:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573911.171088:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573911.171090:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573911.171093:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573911.171095:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573911.171102:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573911.171103:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573911.171105:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573911.171108:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573911.171110:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573911.171114:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573911.171116:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573911.171119:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5c00 x1875440516631808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171124:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573911.171126:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5c00 x1875440516631808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171131:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.171132:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573911.171134:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573911.171137:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573911.171139:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573911.171140:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309271067 00000100:00000001:0.0:1788573911.171142:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309274862 00000100:00000001:0.0:1788573911.171143:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309271067 00000100:00000001:0.0:1788573911.171145:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309274862 00000100:00000001:0.0:1788573911.171146:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573911.171147:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.171149:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573911.171151:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573911/real 1788573911] req@ffff8fc4ac9f5c00 x1875440516631808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573911.171156:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573911.171157:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573911.171158:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573911.171159:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573911.171161:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573911.171162:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573911.171164:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5c00 x1875440516631808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573911.171169:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5c00 x1875440516631808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171176:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573911.171178:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171179:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573911.171180:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573911.171182:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5c00 x1875440516631808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171186:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573911.171189:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5c00 x1875440516631808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171194:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573911.171195:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171196:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573911.171197:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171198:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573911.171200:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573911.171201:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573911.171203:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573911.171205:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573911.171207:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5c00 x1875440516631808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573911.171213:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631808:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573911.171215:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573911.171218:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6300 x1875440516631936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171222:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573911.171223:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573911.171227:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573911.171229:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573911.171230:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573911.171232:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631936:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573911.171234:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573911.171236:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573911.171237:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573911.171238:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573911.171240:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573911.171241:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573911.171244:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573911.171245:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573911.171247:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573911.171249:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516631936, portal 4 00000100:00000001:0.0:1788573911.171251:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573911.171252:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605329664 : -123400104221952 : ffff8fc4ac9f6300) 00000100:00000040:0.0:1788573911.171255:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6300 x1875440516631936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171259:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573911.171261:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573911.171262:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573911.171263:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516631936, offset 0 00000400:00000200:0.0:1788573911.171265:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573911.171268:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573911.171270:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573911.171272:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573911.171274:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573911.171276:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573911.171277:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573911.171279:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546589:546589:256:4294967295] 192.168.202.128@tcp LPNI seq info [546512:546512:8:4294967295] 00000400:00000200:0.0:1788573911.171282:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573911.171285:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573911.171292:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573911.171294:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573911.171296:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573911.171298:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573911.171305:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573911.171306:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573911.171308:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573911.171310:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573911.171312:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573911.171314:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573911.171315:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573911.171317:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6300 x1875440516631936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171322:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573911.171324:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6300 x1875440516631936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171328:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.171329:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573911.171331:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573911.171333:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573911.171335:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573911.171336:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309271207 00000100:00000001:0.0:1788573911.171338:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309274862 00000100:00000001:0.0:1788573911.171339:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309271207 00000100:00000001:0.0:1788573911.171341:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309274862 00000100:00000001:0.0:1788573911.171342:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573911.171344:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.171345:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573911.171347:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573911/real 1788573911] req@ffff8fc4ac9f6300 x1875440516631936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573911.171356:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573911.171357:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573911.171358:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573911.171359:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573911.171360:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573911.171362:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573911.171364:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6300 x1875440516631936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573911.171369:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6300 x1875440516631936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171373:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573911.171374:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171375:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573911.171376:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573911.171378:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6300 x1875440516631936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171382:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573911.171384:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6300 x1875440516631936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171388:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573911.171390:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171391:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573911.171392:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171393:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573911.171394:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573911.171396:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573911.171398:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573911.171399:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573911.171405:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6300 x1875440516631936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573911.171410:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516631936:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573911.171413:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573911.171415:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4380 x1875440516632064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171419:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573911.171420:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573911.171421:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573911.171423:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573911.171424:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573911.171425:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516632064:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573911.171428:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573911.171429:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573911.171430:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573911.171431:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573911.171433:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573911.171435:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573911.171437:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573911.171438:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573911.171440:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573911.171443:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516632064, portal 4 00000100:00000001:0.0:1788573911.171444:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573911.171445:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605321600 : -123400104230016 : ffff8fc4ac9f4380) 00000100:00000040:0.0:1788573911.171448:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4380 x1875440516632064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171452:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573911.171454:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573911.171455:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573911.171457:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516632064, offset 0 00000400:00000200:0.0:1788573911.171463:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573911.171466:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573911.171468:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573911.171470:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573911.171472:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573911.171473:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573911.171475:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573911.171477:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546590:546590:256:4294967295] 192.168.202.128@tcp LPNI seq info [546513:546513:8:4294967295] 00000400:00000200:0.0:1788573911.171480:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573911.171484:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573911.171487:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573911.171489:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573911.171491:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573911.171493:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788573911.171500:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573911.171501:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573911.171503:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573911.171505:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573911.171507:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573911.171508:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573911.171509:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573911.171512:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4380 x1875440516632064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171517:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573911.171518:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4380 x1875440516632064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171523:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.171523:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573911.171525:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573911.171527:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573911.171529:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573911.171534:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309271209 00000100:00000001:0.0:1788573911.171535:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309274862 00000100:00000001:0.0:1788573911.171537:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309271209 00000100:00000001:0.0:1788573911.171538:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309274862 00000100:00000001:0.0:1788573911.171539:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573911.171541:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.171542:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573911.171544:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573911/real 1788573911] req@ffff8fc4ac9f4380 x1875440516632064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573911.171550:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573911.171551:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573911.171552:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573911.171553:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573911.171554:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573911.171555:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573911.171557:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4380 x1875440516632064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573911.171562:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4380 x1875440516632064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171566:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573911.171567:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171568:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573911.171569:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573911.171571:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4380 x1875440516632064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171575:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573911.171578:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4380 x1875440516632064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171582:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573911.171586:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171587:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573911.171588:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171589:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573911.171591:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573911.171593:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573911.171595:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573911.171596:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573911.171598:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4380 x1875440516632064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573911.171603:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516632064:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573911.171605:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171606:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573911.171608:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6d80 x1875440516631680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171613:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573911.171614:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573911.171616:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573911.171617:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573911.171618:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573911.171619:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573911.171621:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573911.171622:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35000. 02000000:00000001:0.0:1788573911.171624:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573911.171625:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573911.171626:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6d80. 00000100:00000001:0.0:1788573911.171628:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573911.171629:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171630:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573911.171632:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5c00 x1875440516631808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171643:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573911.171644:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573911.171645:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573911.171647:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573911.171647:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573911.171648:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573911.171649:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573911.171650:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34c00. 02000000:00000001:0.0:1788573911.171651:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573911.171652:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573911.171653:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5c00. 00000100:00000001:0.0:1788573911.171654:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573911.171655:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171656:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573911.171658:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6300 x1875440516631936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171662:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573911.171662:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573911.171663:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573911.171664:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573911.171665:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573911.171666:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573911.171667:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573911.171667:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35c00. 02000000:00000001:0.0:1788573911.171669:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573911.171669:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573911.171670:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6300. 00000100:00000001:0.0:1788573911.171671:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573911.171672:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171673:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573911.171675:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4380 x1875440516632064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573936 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573911.171678:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573911.171679:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573911.171683:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788573911.171685:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573911.171686:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573911.171687:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573911.171688:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573911.171689:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35400. 02000000:00000001:0.0:1788573911.171690:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573911.171691:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573911.171692:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4380. 00000100:00000001:0.0:1788573911.171693:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573911.171694:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171695:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.171697:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573911.171698:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.171700:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573911.171700:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.680977:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573911.680984:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573911.680988:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573911.680993:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573911.680998:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573911.681005:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.681012:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573911.681016:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.681018:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573911.681043:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573911.681048:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573911.681050:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573911.681054:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573911.681059:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573911.681062:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.681073:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573911.681076:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573911.681079:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573912.385054:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573912.385059:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573912.385061:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573912.704967:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573912.704998:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573912.705005:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573912.705012:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573912.705018:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573912.705027:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573912.705031:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573912.705037:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573912.705041:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573913.728929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573913.728938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573913.728944:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573913.728953:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573913.728960:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573913.728964:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573913.728970:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573913.728974:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573913.728979:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573914.752952:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573914.752956:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573914.752960:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573914.752964:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573914.752968:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573914.752973:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573914.752987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573914.752991:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573914.752996:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573915.008920:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309274862, at 4309274880 00000100:00000010:0.0:1788573915.008938:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573915.457004:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573915.457008:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573915.457011:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573915.776993:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573915.777000:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573915.777004:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573915.777011:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573915.777017:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573915.777020:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573915.777043:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573915.777048:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573915.777053:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573916.288984:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573916.288994:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573916.288997:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573916.289001:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573916.289006:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573916.289014:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14603 00000400:00000200:0.0:1788573916.289024:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573916.289030:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573916.289033:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573916.289036:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289039:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573916.289041:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289043:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573916.289047:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573916.289049:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573916.289051:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573916.289054:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573916.289056:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573916.289058:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573916.289061:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289063:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573916.289064:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289065:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573916.289068:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573916.289071:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289072:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573916.289075:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573916.289078:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573916.289106:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.289108:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573916.289110:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573916.289120:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788573916.289123:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573916.289127:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573916.289128:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.289131:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573916.289134:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573916.289148:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573916.289157:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573916.289165:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573916.289176:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516632192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573916.289185:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573916.289189:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573916.289194:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516632192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573916.289216:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573916.289219:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573916.289222:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573916.289223:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573916.289224:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573916.289227:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573916.289229:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14603 00000400:00000200:0.0:1788573916.289233:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573916.289239:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573916.289241:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573916.289244:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289246:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573916.289249:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289258:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573916.289263:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573916.289265:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573916.289269:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573916.289272:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573916.289274:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573916.289276:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573916.289282:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289284:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573916.289286:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289291:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573916.289298:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573916.289305:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289313:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573916.289320:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573916.289337:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573916.289348:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.289355:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573916.289360:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573916.289371:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.289374:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573916.289380:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788573916.289386:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573916.289391:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573916.289394:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.289397:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573916.289403:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573916.289416:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573916.289422:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573916.289438:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573916.289447:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516632320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573916.289477:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573916.289479:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573916.289484:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516632320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573916.289495:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573916.289500:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573916.289513:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573916.289522:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573916.289531:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573916.289540:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573916.289547:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14603 00000400:00000200:0.0:1788573916.289559:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573916.289570:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573916.289573:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573916.289578:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289582:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573916.289585:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289588:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573916.289590:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573916.289591:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573916.289593:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573916.289594:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573916.289596:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573916.289598:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573916.289608:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289611:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573916.289622:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289628:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573916.289639:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573916.289647:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289652:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573916.289656:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573916.289658:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573916.289660:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.289663:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573916.289666:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573916.289676:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.289678:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573916.289683:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788573916.289694:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573916.289702:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573916.289708:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.289712:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573916.289714:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573916.289719:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573916.289730:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573916.289739:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573916.289748:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516632448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573916.289764:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573916.289770:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573916.289786:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516632448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573916.289893:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573916.289899:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573916.289912:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573916.289915:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573916.289919:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573916.289924:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573916.289936:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14603 00000400:00000200:0.0:1788573916.289946:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573916.289951:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573916.289959:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573916.289962:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289963:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573916.289965:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289967:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573916.289968:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573916.289969:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573916.289971:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573916.289972:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573916.289974:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573916.289976:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573916.289981:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289987:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573916.289991:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.289998:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573916.290006:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573916.290012:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573916.290019:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573916.290025:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573916.290028:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573916.290031:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.290033:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573916.290036:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573916.290040:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.290046:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573916.290051:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788573916.290055:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573916.290064:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573916.290072:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.290078:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573916.290080:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573916.290082:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573916.290088:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573916.290096:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573916.290104:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516632576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573916.290121:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573916.290125:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573916.290131:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516632576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573916.290144:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573916.290150:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573916.290156:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573916.290158:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573916.290161:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573916.290168:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573916.290174:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14603 00000400:00000200:0.0:1788573916.290185:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573916.290194:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573916.290199:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573916.290207:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.290213:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573916.290221:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.290226:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573916.290235:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573916.290238:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573916.290243:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573916.290247:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573916.290253:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573916.290265:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573916.290272:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.290275:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573916.290282:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573916.290286:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573916.290292:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573916.290295:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573916.290298:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573916.290303:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573916.290306:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573916.290312:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.290317:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573916.290320:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573916.290326:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.290330:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573916.290336:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788573916.290342:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573916.290345:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573916.290350:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573916.290352:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573916.290356:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573916.290358:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573916.290364:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573916.290371:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573916.290379:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516632704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573916.290390:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573916.290395:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573916.290401:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516632704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573916.290413:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573916.290429:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14613) 00000100:00000001:0.0:1788573916.800984:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573916.800996:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573916.801003:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573916.801014:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573916.801028:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573916.801036:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573916.801059:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573916.801065:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573916.801072:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573917.505010:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573917.505013:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573917.505016:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573917.824938:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573917.824944:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573917.824950:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573917.824962:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573917.824972:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573917.824979:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573917.824998:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573917.825002:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573917.825006:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573918.848922:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573918.848930:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573918.848933:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573918.848939:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573918.848953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573918.848959:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573918.848973:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573918.848978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573918.848985:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573919.361137:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788573919.872939:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573919.872947:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573919.872951:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573919.872958:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573919.872963:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573919.872967:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573919.872998:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573919.873001:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573919.873005:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573919.873039:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573919.873042:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573919.873043:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573919.873046:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573919.873048:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573919.873050:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573919.873052:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573919.873053:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573919.873055:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573920.128939:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309279982, at 4309280000 00000100:00000010:0.0:1788573920.128968:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f00. 00000020:00000001:0.0:1788573920.577027:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573920.577030:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573920.577033:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573920.896901:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573920.896908:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573920.896912:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573920.896920:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573920.896928:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573920.896934:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573920.896943:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573920.896948:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573920.896954:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573921.408926:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573921.408940:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573921.408943:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573921.408947:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573921.408951:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573921.408958:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14608 00000400:00000200:0.0:1788573921.408966:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573921.408971:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573921.408974:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573921.408992:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.408995:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573921.408998:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409000:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573921.409003:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573921.409004:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573921.409006:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573921.409009:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573921.409011:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573921.409012:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573921.409014:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409015:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573921.409017:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409018:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573921.409020:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573921.409023:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409025:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573921.409027:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573921.409030:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573921.409032:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409033:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573921.409035:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573921.409040:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573921.409043:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573921.409047:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573921.409048:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409051:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573921.409055:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573921.409060:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573921.409067:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573921.409075:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573921.409085:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516632832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573921.409100:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573921.409104:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573921.409108:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516632832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573921.409126:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573921.409129:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573921.409132:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573921.409133:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573921.409134:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573921.409136:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573921.409138:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14608 00000400:00000200:0.0:1788573921.409141:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573921.409144:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573921.409146:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573921.409148:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409150:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573921.409152:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409154:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573921.409156:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573921.409158:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573921.409159:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573921.409160:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573921.409162:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573921.409164:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573921.409168:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409170:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573921.409175:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409177:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573921.409186:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573921.409190:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409197:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573921.409201:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573921.409209:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573921.409214:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409219:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573921.409225:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573921.409230:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409235:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573921.409241:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573921.409246:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573921.409251:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573921.409253:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409256:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573921.409258:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573921.409261:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573921.409266:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573921.409271:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573921.409276:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516632960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573921.409287:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573921.409291:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573921.409303:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516632960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573921.409317:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573921.409320:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573921.409326:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573921.409327:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573921.409329:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573921.409331:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573921.409338:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14608 00000400:00000200:0.0:1788573921.409345:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573921.409349:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573921.409352:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573921.409357:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409359:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573921.409363:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409367:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573921.409371:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573921.409374:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573921.409377:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573921.409380:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573921.409384:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573921.409388:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573921.409392:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409395:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573921.409396:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409398:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573921.409400:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573921.409401:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409403:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573921.409407:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573921.409410:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573921.409411:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409415:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573921.409417:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573921.409419:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409421:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573921.409426:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573921.409429:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573921.409431:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573921.409437:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409441:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573921.409443:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573921.409445:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573921.409450:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573921.409456:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573921.409462:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516633088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573921.409474:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573921.409477:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573921.409485:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516633088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573921.409496:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573921.409501:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573921.409505:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573921.409508:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573921.409509:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573921.409514:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573921.409517:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14608 00000400:00000200:0.0:1788573921.409533:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573921.409538:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573921.409541:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573921.409543:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409545:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573921.409549:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409551:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573921.409554:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573921.409556:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573921.409558:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573921.409561:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573921.409564:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573921.409573:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573921.409585:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409593:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573921.409597:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409600:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573921.409606:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573921.409608:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409611:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573921.409613:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573921.409619:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573921.409625:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409630:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573921.409633:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573921.409637:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409641:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573921.409647:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573921.409652:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573921.409657:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573921.409660:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409665:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573921.409667:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573921.409670:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573921.409676:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573921.409682:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573921.409690:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516633216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573921.409701:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573921.409704:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573921.409709:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516633216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573921.409720:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573921.409727:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573921.409729:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573921.409731:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573921.409732:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573921.409735:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573921.409739:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14608 00000400:00000200:0.0:1788573921.409747:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573921.409750:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573921.409754:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573921.409757:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409758:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573921.409761:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409765:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573921.409767:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573921.409768:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573921.409770:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573921.409772:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573921.409775:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573921.409778:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573921.409780:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409785:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573921.409787:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409792:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573921.409795:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573921.409798:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573921.409802:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573921.409806:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573921.409830:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573921.409834:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409837:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573921.409844:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573921.409849:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409853:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573921.409857:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573921.409862:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573921.409866:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573921.409869:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573921.409871:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573921.409874:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573921.409876:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573921.409882:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573921.409899:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573921.409910:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516633344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573921.409925:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573921.409930:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573921.409939:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516633344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573921.409960:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573921.409964:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14618) 00000100:00000001:0.0:1788573921.920919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573921.920925:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573921.920930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573921.920937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573921.920943:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573921.920951:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573921.920968:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573921.920971:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573921.920975:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573922.625061:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573922.625065:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573922.625068:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573922.944890:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573922.944895:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573922.944926:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573922.944931:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573922.944937:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573922.944942:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573922.944949:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573922.944953:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573922.944958:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573922.944989:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573922.944994:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573922.944996:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573922.944999:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573922.945002:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573922.945005:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573922.945008:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573922.945009:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573922.945012:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573923.969072:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573923.969075:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573923.969077:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573923.969079:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573923.969082:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573923.969084:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573923.969086:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573923.969090:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573923.969092:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573924.993141:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573924.993144:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573924.993146:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573924.993149:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573924.993151:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573924.993153:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573924.993209:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573924.993212:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573924.993214:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573925.248910:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309285101, at 4309285120 00000100:00000010:0.0:1788573925.248945:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7120. 00000020:00000001:0.0:1788573925.697011:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573925.697020:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573925.697023:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573926.016894:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573926.016903:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573926.016908:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573926.016917:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573926.016926:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573926.016932:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573926.016962:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573926.016966:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573926.016971:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573926.017005:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573926.017008:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573926.017009:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573926.017011:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573926.017013:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573926.017015:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573926.017017:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573926.017018:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573926.017019:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573927.040928:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573927.040933:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573927.040936:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573927.040940:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573927.040946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573927.040960:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573927.040972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573927.040978:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573927.040983:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573927.041013:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573927.041017:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573927.041020:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573927.041027:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573927.041032:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573927.041036:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573927.041039:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573927.041042:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573927.041045:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573927.745006:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573927.745011:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573927.745015:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573928.064927:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573928.064936:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573928.064941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573928.064950:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573928.064959:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573928.064977:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573928.064984:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573928.064987:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573928.064991:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573928.065014:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573928.065017:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573928.065021:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573928.065023:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573928.065028:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573928.065031:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573928.065034:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573928.065035:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573928.065038:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573929.088935:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573929.088939:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573929.088942:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573929.088949:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573929.088952:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573929.088956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573929.088998:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573929.089002:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573929.089006:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573930.112938:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573930.112943:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573930.112948:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573930.112952:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573930.112956:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573930.112960:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573930.112971:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573930.112975:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573930.113012:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573930.368898:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309290221, at 4309290240 00000100:00000010:0.0:1788573930.368911:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7000. 00000020:00000001:0.0:1788573930.817023:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573930.817028:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573930.817031:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573932.160943:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573932.160964:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573932.160968:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573932.160974:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573932.160979:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573932.160985:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573932.160993:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573932.160998:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573932.161003:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573932.864991:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573932.864996:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573932.864999:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573933.184946:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573933.184956:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573933.184962:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573933.184973:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573933.184978:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573933.184987:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573933.185033:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573933.185037:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573933.185040:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573934.208968:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573934.208980:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573934.208986:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573934.209004:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573934.209026:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573934.209048:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573934.209059:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573934.209064:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573934.209081:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573935.232936:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573935.232965:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573935.232970:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573935.232978:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573935.232985:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573935.232989:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573935.233004:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573935.233007:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573935.233009:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573935.488909:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309295341, at 4309295360 00000100:00000010:0.0:1788573935.488928:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7600. 00000020:00000001:0.0:1788573935.937097:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573935.937101:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573935.937104:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573936.256929:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573936.256941:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573936.256944:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573936.256951:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573936.256956:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573936.256964:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573936.256971:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573936.256976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573936.256980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573937.280906:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573937.280912:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573937.280917:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573937.280922:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573937.280930:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573937.280936:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573937.280954:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573937.280962:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573937.280966:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573937.985022:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573937.985026:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573937.985029:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573938.304882:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573938.304890:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573938.304896:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573938.304920:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573938.304928:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573938.304934:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573938.304950:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573938.304955:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573938.304958:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573939.328921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573939.328926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573939.328931:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573939.328945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573939.328951:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573939.328954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573939.328965:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573939.328969:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573939.328976:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573940.352918:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573940.352925:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573940.352928:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573940.352937:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573940.352951:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573940.352957:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573940.352971:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573940.352974:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573940.352977:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573940.608929:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309300462, at 4309300480 00000100:00000010:0.0:1788573940.608947:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ba0. 00000020:00000001:0.0:1788573941.057027:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573941.057031:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573941.057033:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573941.376930:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573941.376936:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573941.376941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573941.376949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573941.376956:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573941.376964:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573941.376973:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573941.376978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573941.377014:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573942.400949:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573942.400953:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573942.400957:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573942.400961:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573942.400965:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573942.400967:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573942.400969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573942.400971:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573942.400973:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573943.104971:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573943.104975:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573943.104980:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573943.424910:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573943.424919:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573943.424922:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573943.424928:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573943.424933:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573943.424939:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573943.424945:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573943.424948:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573943.424955:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573943.425025:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573943.425029:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573943.425030:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573943.425032:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573943.425034:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573943.425035:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573943.425187:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573943.425191:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573943.425195:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573944.448978:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573944.448982:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573944.448984:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573944.448986:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573944.448992:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573944.448994:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573944.449846:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573944.449866:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573944.449870:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573945.472926:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573945.472929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573945.472933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573945.472940:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573945.472945:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573945.472949:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573945.472983:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573945.472987:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573945.472993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573945.728923:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309305581, at 4309305600 00000100:00000010:0.0:1788573945.728944:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7420. 00000020:00000001:0.0:1788573946.177024:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573946.177027:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573946.177029:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573946.496918:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573946.496928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573946.496932:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573946.496938:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573946.496945:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573946.496950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573946.496954:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573946.496958:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573946.496966:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573947.008924:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573947.008936:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573947.008939:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573947.008944:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573947.008949:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573947.008957:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14634 00000400:00000200:0.0:1788573947.008967:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573947.008975:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573947.008979:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573947.008983:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009005:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573947.009008:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009011:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573947.009025:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573947.009027:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573947.009030:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573947.009034:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573947.009037:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573947.009040:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573947.009044:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009047:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573947.009049:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009051:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573947.009054:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573947.009058:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009061:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573947.009064:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573947.009068:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573947.009071:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.009074:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573947.009077:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573947.009084:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573947.009088:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573947.009093:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573947.009095:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.009098:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573947.009101:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573947.009106:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573947.009114:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573947.009121:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573947.009131:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516636032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573947.009155:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573947.009159:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573947.009165:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516636032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573947.009198:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573947.009203:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573947.009207:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573947.009209:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573947.009211:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573947.009215:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573947.009219:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14634 00000400:00000200:0.0:1788573947.009226:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573947.009235:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573947.009239:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573947.009246:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009252:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573947.009258:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009263:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573947.009268:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573947.009272:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573947.009277:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573947.009282:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573947.009287:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573947.009293:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573947.009300:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009306:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573947.009311:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009316:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573947.009331:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573947.009338:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009343:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573947.009347:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573947.009354:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573947.009360:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.009364:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573947.009371:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573947.009377:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.009383:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573947.009391:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573947.009396:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573947.009399:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573947.009400:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.009404:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573947.009408:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573947.009412:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573947.009419:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573947.009424:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573947.009432:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516636160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573947.009449:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573947.009462:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573947.009474:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516636160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573947.009499:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573947.009504:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573947.009512:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573947.009516:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573947.009519:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573947.009525:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573947.009538:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14634 00000400:00000200:0.0:1788573947.009550:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573947.009561:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573947.009565:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573947.009571:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009576:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573947.009582:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009588:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573947.009594:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573947.009598:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573947.009604:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573947.009607:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573947.009613:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573947.009617:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573947.009626:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009630:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573947.009634:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009638:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573947.009644:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573947.009653:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009660:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573947.009663:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573947.009669:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573947.009672:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.009677:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573947.009682:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573947.009687:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.009691:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573947.009696:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573947.009701:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573947.009706:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573947.009716:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.009721:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573947.009725:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573947.009729:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573947.009736:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573947.009745:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573947.009755:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516636288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573947.009776:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573947.009781:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573947.009791:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516636288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573947.009864:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573947.009870:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573947.009878:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573947.009883:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573947.009886:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573947.009893:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573947.009901:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14634 00000400:00000200:0.0:1788573947.009912:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573947.009921:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573947.009928:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573947.009937:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009942:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573947.009948:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.009953:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573947.009959:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573947.009964:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573947.009969:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573947.009972:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573947.009977:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573947.009989:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573947.009997:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.010004:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573947.010011:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.010017:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573947.010023:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573947.010031:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573947.010038:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573947.010045:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573947.010051:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573947.010056:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.010062:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573947.010067:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573947.010073:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.010076:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573947.010082:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573947.010088:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573947.010093:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573947.010097:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.010102:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573947.010105:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573947.010110:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573947.010119:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573947.010127:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573947.010137:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516636416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573947.010155:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573947.010160:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573947.010170:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516636416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573947.010204:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573947.010215:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573947.010221:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573947.010226:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573947.010229:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573947.010233:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573947.010239:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14634 00000400:00000200:0.0:1788573947.010246:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573947.010254:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573947.010257:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573947.010263:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.010269:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573947.010273:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.010277:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573947.010282:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573947.010285:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573947.010287:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573947.010290:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573947.010295:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573947.010299:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573947.010303:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.010306:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573947.010311:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573947.010315:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573947.010319:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573947.010325:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573947.010330:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573947.010333:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573947.010336:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573947.010339:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.010344:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573947.010350:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573947.010352:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.010355:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573947.010359:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573947.010364:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573947.010368:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573947.010371:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573947.010374:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573947.010377:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573947.010381:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573947.010386:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788573947.010391:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573947.010399:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516636544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573947.010413:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573947.010417:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573947.010422:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516636544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573947.010434:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573947.010439:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14644) 00000100:00000001:0.0:1788573947.520921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573947.520927:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573947.520932:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573947.520939:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573947.520945:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573947.520950:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573947.520961:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573947.520965:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573947.520971:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573948.225021:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573948.225024:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573948.225027:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573948.544918:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573948.544926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573948.544951:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573948.544961:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573948.544969:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573948.544973:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573948.545002:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573948.545006:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573948.545010:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573949.569070:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573949.569077:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573949.569082:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573949.569090:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573949.569099:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573949.569108:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573949.569120:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573949.569124:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573949.569128:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573950.592967:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573950.592976:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573950.592982:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573950.592995:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573950.593008:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573950.593020:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573950.593033:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573950.593042:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573950.593046:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573950.593072:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573950.593076:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573950.593079:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573950.593084:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573950.593088:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573950.593092:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573950.593096:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573950.593100:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573950.593107:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573950.848957:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309310701, at 4309310720 00000100:00000010:0.0:1788573950.848975:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea75a0. 00000020:00000001:0.0:1788573951.297051:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573951.297075:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573951.297078:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573951.616918:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573951.616925:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573951.616930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573951.616937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573951.616943:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573951.616949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573951.616954:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573951.616957:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573951.616962:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573952.640925:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573952.640930:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573952.640934:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573952.640940:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573952.640948:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573952.640955:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573952.640973:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573952.640979:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573952.640987:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573953.345030:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573953.345034:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573953.345037:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573953.664915:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573953.664924:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573953.664930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573953.664941:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573953.664955:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573953.664965:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573953.664991:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573953.664995:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573953.665003:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573953.665051:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573953.665055:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573953.665056:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573953.665059:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573953.665061:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573953.665063:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573953.665081:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573953.665083:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573953.665085:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573954.689009:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573954.689019:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573954.689024:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573954.689033:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573954.689041:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573954.689051:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573954.689068:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573954.689074:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573954.689081:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573955.712891:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573955.712900:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573955.712904:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573955.712911:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573955.712918:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573955.712923:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573955.712946:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573955.712951:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573955.712956:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573955.968879:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309315821, at 4309315840 00000100:00000010:0.0:1788573955.968901:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea73c0. 00000020:00000001:0.0:1788573956.416985:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573956.416988:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573956.416990:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573956.736919:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573956.736926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573956.736930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573956.736936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573956.736951:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573956.736958:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573956.736991:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573956.736994:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573956.736997:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573957.249126:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788573957.249204:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573957.249215:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573957.249218:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573957.249254:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516637312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.249283:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573957.249330:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573957.249335:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573957.249340:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573957.249346:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573957.249364:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516637312:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573957.249371:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573957.249376:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573957.249379:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573957.249382:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573957.249388:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573957.249392:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573957.249399:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573957.249405:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573957.249415:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573957.249426:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516637312, portal 25 00000100:00000001:0.0:1788573957.249432:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573957.249437:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788573957.249450:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516637312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.249472:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573957.249480:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573957.249485:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573957.249490:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516637312, offset 0 00000400:00000200:0.0:1788573957.249501:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573957.249530:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573957.249540:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573957.249570:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573957.249594:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573957.249606:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573957.249612:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573957.249619:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546631:546631:256:4294967295] 192.168.202.128@tcp LPNI seq info [546554:546554:8:4294967295] 00000400:00000200:0.0:1788573957.249628:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573957.249644:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573957.249654:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573957.249660:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573957.249666:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573957.249671:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573957.249697:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573957.249701:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573957.249707:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573957.249715:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573957.249721:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573957.249724:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573957.249731:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573957.249737:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516637312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.249753:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573957.249760:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516637312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.249778:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.249783:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573957.249790:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573957.249804:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573957.249812:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309320941 00000100:00000010:0.0:1788573957.249848:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573957.249857:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573957.249863:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309320942 00000100:00000010:0.0:1788573957.249871:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50b40. 00000100:00000001:0.0:1788573957.249885:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573957.249892:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573957.249899:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.249905:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573957.249914:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573957/real 1788573957] req@ffff8fc4901a7100 x1875440516637312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573957.249936:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573957.249940:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573957.249944:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573957.249949:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573957.249954:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573957.249958:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573957.249966:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516637312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573957.249980:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516637312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.249990:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573957.249992:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.249997:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573957.249999:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573957.250002:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516637312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250013:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573957.250020:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516637312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250030:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573957.250034:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250038:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573957.250040:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250044:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573957.250052:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573957.250056:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573957.250062:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573957.250066:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573957.250073:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516637312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573957.250090:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516637312:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573957.250095:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250098:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573957.250103:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516637312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250116:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573957.250120:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573957.250122:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573957.250126:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573957.250129:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573957.250132:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573957.250135:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573957.250138:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788573957.250155:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573957.250158:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573957.250162:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788573957.250166:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573957.250169:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250173:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250182:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573957.250185:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.250187:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573957.250189:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573957.250191:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573957.250196:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516637440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250211:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573957.250212:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573957.250214:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573957.250216:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573957.250217:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573957.250220:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516637440:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573957.250222:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573957.250224:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573957.250225:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573957.250226:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573957.250228:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573957.250229:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573957.250233:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573957.250235:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573957.250237:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573957.250239:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516637440, portal 10 00000100:00000001:0.0:1788573957.250242:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573957.250243:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788573957.250250:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516637440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250257:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573957.250259:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573957.250262:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573957.250265:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516637440, offset 0 00000400:00000200:0.0:1788573957.250269:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573957.250274:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573957.250278:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573957.250282:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573957.250285:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573957.250287:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573957.250290:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573957.250311:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546632:546632:256:4294967295] 192.168.202.128@tcp LPNI seq info [546555:546555:8:4294967295] 00000400:00000200:0.0:1788573957.250315:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573957.250320:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573957.250323:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573957.250326:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573957.250329:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573957.250331:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573957.250338:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573957.250339:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573957.250341:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573957.250344:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573957.250346:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573957.250347:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573957.250349:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573957.250352:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516637440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250358:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573957.250360:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516637440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250364:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.250365:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573957.250368:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573957.250371:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573957.250373:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573957.250375:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309317139 00000100:00000001:0.0:1788573957.250377:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309320941 00000100:00000001:0.0:1788573957.250378:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309317139 00000100:00000001:0.0:1788573957.250379:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309320942 00000100:00000001:0.0:1788573957.250381:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573957.250386:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.250387:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573957.250390:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573957/real 1788573957] req@ffff8fc4901a5c00 x1875440516637440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573957.250396:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573957.250397:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573957.250398:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573957.250400:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573957.250401:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573957.250402:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573957.250404:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516637440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573957.250410:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516637440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250414:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573957.250415:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250416:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573957.250417:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573957.250419:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516637440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250423:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573957.250425:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516637440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250430:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573957.250431:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250432:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573957.250433:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250434:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573957.250435:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573957.250437:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573957.250442:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573957.250443:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573957.250446:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516637440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573957.250451:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516637440:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573957.250453:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573957.250455:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516637568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250460:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573957.250461:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573957.250463:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573957.250464:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573957.250466:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573957.250467:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516637568:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573957.250469:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573957.250470:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573957.250471:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573957.250473:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573957.250475:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573957.250476:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573957.250479:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573957.250480:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573957.250482:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573957.250484:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516637568, portal 10 00000100:00000001:0.0:1788573957.250486:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573957.250487:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788573957.250491:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516637568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250495:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573957.250500:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573957.250501:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573957.250503:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516637568, offset 0 00000400:00000200:0.0:1788573957.250505:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573957.250508:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573957.250511:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573957.250513:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573957.250515:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573957.250516:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573957.250518:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573957.250520:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546633:546633:256:4294967295] 192.168.202.128@tcp LPNI seq info [546556:546556:8:4294967295] 00000400:00000200:0.0:1788573957.250523:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573957.250527:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573957.250530:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573957.250532:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573957.250535:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573957.250536:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573957.250543:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573957.250545:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573957.250547:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573957.250549:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573957.250551:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573957.250552:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573957.250554:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573957.250556:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516637568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250561:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573957.250563:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516637568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250568:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.250572:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573957.250574:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573957.250576:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573957.250579:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573957.250580:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309317146 00000100:00000001:0.0:1788573957.250582:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309320941 00000100:00000001:0.0:1788573957.250583:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309317146 00000100:00000001:0.0:1788573957.250585:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309320942 00000100:00000001:0.0:1788573957.250586:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573957.250587:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.250588:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573957.250591:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573957/real 1788573957] req@ffff8fc4901a5180 x1875440516637568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573957.250595:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573957.250596:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573957.250597:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573957.250599:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573957.250600:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573957.250601:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573957.250603:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516637568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573957.250608:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516637568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250612:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573957.250613:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250614:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573957.250615:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573957.250617:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516637568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250621:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573957.250626:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516637568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250631:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573957.250633:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250634:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573957.250635:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250636:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573957.250638:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573957.250639:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573957.250641:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573957.250642:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573957.250645:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516637568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573957.250650:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516637568:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573957.250652:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573957.250654:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516637696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250658:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573957.250659:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573957.250660:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573957.250662:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573957.250663:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573957.250664:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516637696:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573957.250666:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573957.250667:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573957.250668:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573957.250670:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573957.250672:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573957.250673:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573957.250678:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573957.250680:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573957.250682:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573957.250684:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516637696, portal 4 00000100:00000001:0.0:1788573957.250686:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573957.250686:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788573957.250690:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516637696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250695:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573957.250696:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573957.250697:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573957.250699:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516637696, offset 0 00000400:00000200:0.0:1788573957.250701:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573957.250704:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573957.250706:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573957.250708:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573957.250709:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573957.250711:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573957.250712:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573957.250714:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546634:546634:256:4294967295] 192.168.202.128@tcp LPNI seq info [546557:546557:8:4294967295] 00000400:00000200:0.0:1788573957.250717:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573957.250721:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573957.250724:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573957.250725:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573957.250728:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573957.250729:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573957.250737:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573957.250738:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573957.250740:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573957.250742:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573957.250744:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573957.250749:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573957.250751:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573957.250753:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516637696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250758:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573957.250760:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516637696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250764:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.250766:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573957.250768:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573957.250770:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573957.250772:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573957.250773:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309317286 00000100:00000001:0.0:1788573957.250775:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309320941 00000100:00000001:0.0:1788573957.250776:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309317286 00000100:00000001:0.0:1788573957.250778:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309320942 00000100:00000001:0.0:1788573957.250779:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573957.250781:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.250782:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573957.250784:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573957/real 1788573957] req@ffff8fc4901a7800 x1875440516637696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573957.250788:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573957.250789:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573957.250790:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573957.250791:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573957.250793:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573957.250794:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573957.250797:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516637696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573957.250806:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516637696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250837:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573957.250839:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250841:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573957.250843:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573957.250848:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516637696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250853:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573957.250855:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516637696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250863:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573957.250865:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250867:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573957.250868:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.250870:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573957.250872:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573957.250875:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573957.250878:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573957.250881:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573957.250885:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516637696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573957.250902:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516637696:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573957.250906:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573957.250910:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516637824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250914:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573957.250916:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573957.250922:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573957.250923:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573957.250924:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573957.250926:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516637824:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573957.250929:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573957.250930:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573957.250931:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573957.250932:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573957.250934:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573957.250936:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573957.250938:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573957.250940:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573957.250942:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573957.250944:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516637824, portal 4 00000100:00000001:0.0:1788573957.250946:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573957.250947:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788573957.250950:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516637824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.250954:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573957.250956:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573957.250957:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573957.250959:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516637824, offset 0 00000400:00000200:0.0:1788573957.250961:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573957.250964:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573957.250966:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573957.250968:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573957.250970:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573957.250971:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573957.250973:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573957.250975:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546635:546635:256:4294967295] 192.168.202.128@tcp LPNI seq info [546558:546558:8:4294967295] 00000400:00000200:0.0:1788573957.250978:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573957.250982:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573957.250989:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573957.250991:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573957.250993:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573957.250995:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788573957.251002:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573957.251003:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573957.251005:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573957.251008:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573957.251010:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573957.251011:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573957.251013:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573957.251015:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516637824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.251020:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573957.251022:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516637824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.251027:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.251027:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573957.251029:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573957.251031:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573957.251033:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573957.251034:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309317289 00000100:00000001:0.0:1788573957.251036:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309320941 00000100:00000001:0.0:1788573957.251038:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309317289 00000100:00000001:0.0:1788573957.251039:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309320942 00000100:00000001:0.0:1788573957.251040:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573957.251041:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.251043:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573957.251045:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573957/real 1788573957] req@ffff8fc4901a4a80 x1875440516637824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573957.251053:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573957.251054:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573957.251056:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573957.251057:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573957.251058:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573957.251059:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573957.251062:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516637824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573957.251067:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516637824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.251072:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573957.251072:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.251074:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573957.251074:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573957.251077:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516637824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.251080:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573957.251083:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516637824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.251087:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573957.251088:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.251089:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573957.251090:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.251091:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573957.251092:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573957.251094:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573957.251096:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573957.251097:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573957.251102:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516637824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573957.251107:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516637824:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573957.251109:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.251111:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573957.251113:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516637440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.251116:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573957.251118:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573957.251119:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573957.251121:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573957.251122:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573957.251123:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573957.251124:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573957.251125:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6400. 02000000:00000001:0.0:1788573957.251127:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573957.251128:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573957.251129:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788573957.251132:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573957.251133:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.251135:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573957.251137:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516637568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.251141:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573957.251142:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573957.251167:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573957.251169:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573957.251170:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573957.251171:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573957.251172:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573957.251173:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7000. 02000000:00000001:0.0:1788573957.251174:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573957.251179:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573957.251180:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788573957.251182:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573957.251183:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.251184:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573957.251186:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516637696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.251191:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573957.251192:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573957.251193:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573957.251194:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573957.251195:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573957.251196:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573957.251197:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573957.251198:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788573957.251200:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573957.251200:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573957.251201:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788573957.251203:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573957.251203:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.251204:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573957.251206:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516637824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573957.251210:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573957.251211:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573957.251212:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573957.251215:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573957.251216:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573957.251217:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573957.251218:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573957.251219:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788573957.251220:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573957.251222:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573957.251223:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788573957.251226:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573957.251227:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.251232:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.251235:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573957.251236:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.251237:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573957.251238:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573957.760890:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573957.760898:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573957.760901:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573957.760907:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573957.760913:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573957.760919:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.760925:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573957.760930:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573957.760935:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573958.465008:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573958.465011:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573958.465014:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573958.784966:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573958.784973:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573958.784976:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573958.784980:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573958.784987:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573958.784992:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573958.784994:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573958.784996:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573958.784998:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573959.808919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573959.808926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573959.808931:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573959.808939:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573959.808949:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573959.808955:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573959.808963:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573959.808968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573959.808972:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573959.809001:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573959.809005:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573959.809006:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573959.809024:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573959.809026:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573959.809028:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573959.809030:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573959.809031:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573959.809034:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573960.832918:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573960.832925:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573960.832930:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573960.832938:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573960.832944:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573960.832951:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573960.832957:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573960.832961:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573960.832967:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573961.088933:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309320941, at 4309320960 00000100:00000010:0.0:1788573961.088955:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573961.537022:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573961.537025:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573961.537029:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573961.856956:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573961.856967:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573961.856970:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573961.856974:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573961.856979:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573961.856985:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573961.857009:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573961.857013:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573961.857016:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.369147:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573962.369153:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573962.369159:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573962.369169:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516637952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.369189:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573962.369192:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573962.369208:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573962.369214:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573962.369222:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573962.369230:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516637952:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573962.369238:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573962.369242:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573962.369246:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573962.369251:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573962.369258:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573962.369264:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573962.369276:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573962.369281:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573962.369293:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573962.369303:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516637952, portal 25 00000100:00000001:0.0:1788573962.369309:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573962.369314:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788573962.369328:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516637952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.369351:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573962.369373:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573962.369378:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573962.369385:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516637952, offset 0 00000400:00000200:0.0:1788573962.369396:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573962.369406:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573962.369417:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573962.369426:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573962.369435:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573962.369442:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573962.369450:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573962.369458:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546636:546636:256:4294967295] 192.168.202.128@tcp LPNI seq info [546559:546559:8:4294967295] 00000400:00000200:0.0:1788573962.369467:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573962.369484:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573962.369504:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573962.369513:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573962.369525:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573962.369532:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573962.369565:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573962.369571:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573962.369580:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573962.369588:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573962.369597:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573962.369602:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573962.369610:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573962.369620:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516637952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.369643:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573962.369652:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516637952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.369673:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.369678:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573962.369686:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573962.369696:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573962.369702:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309326061 00000100:00000010:0.0:1788573962.369711:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573962.369721:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573962.369731:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309326061 00000100:00000010:0.0:1788573962.369740:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50b40. 00000100:00000001:0.0:1788573962.369748:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573962.369754:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573962.369769:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.369776:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573962.369785:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573962/real 1788573962] req@ffff8fc4ac822a00 x1875440516637952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573962.369847:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573962.369850:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573962.369854:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573962.369860:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573962.369865:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573962.369871:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573962.369879:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516637952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573962.369900:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516637952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.369920:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573962.369923:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.369929:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573962.369935:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573962.369945:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516637952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.369966:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573962.369977:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516637952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.369995:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573962.370001:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.370004:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573962.370009:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.370014:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573962.370019:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573962.370027:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573962.370036:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573962.370042:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573962.370055:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516637952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573962.370084:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516637952:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573962.370094:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.370099:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573962.370106:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516637952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370124:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573962.370129:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573962.370134:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573962.370142:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573962.370146:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573962.370150:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573962.370155:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573962.370160:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788573962.370166:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573962.370170:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573962.370175:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788573962.370180:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573962.370184:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.370188:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.370197:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573962.370202:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.370206:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573962.370209:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573962.370212:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573962.370218:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516638080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370228:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573962.370230:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573962.370233:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573962.370235:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573962.370238:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573962.370241:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638080:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573962.370253:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573962.370255:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573962.370257:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573962.370259:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573962.370262:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573962.370265:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573962.370269:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573962.370272:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573962.370276:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573962.370280:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516638080, portal 10 00000100:00000001:0.0:1788573962.370282:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573962.370284:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788573962.370291:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516638080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370299:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573962.370302:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573962.370305:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573962.370307:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516638080, offset 0 00000400:00000200:0.0:1788573962.370312:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573962.370316:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573962.370320:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573962.370324:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573962.370327:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573962.370330:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573962.370333:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573962.370336:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546637:546637:256:4294967295] 192.168.202.128@tcp LPNI seq info [546560:546560:8:4294967295] 00000400:00000200:0.0:1788573962.370342:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573962.370349:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573962.370354:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573962.370373:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573962.370378:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573962.370386:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573962.370399:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573962.370401:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573962.370405:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573962.370409:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573962.370413:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573962.370415:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573962.370418:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573962.370422:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516638080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370431:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573962.370435:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516638080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370443:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.370445:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573962.370448:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573962.370452:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573962.370456:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573962.370458:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309322259 00000100:00000001:0.0:1788573962.370462:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309326061 00000100:00000001:0.0:1788573962.370464:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309322259 00000100:00000001:0.0:1788573962.370467:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309326061 00000100:00000001:0.0:1788573962.370469:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573962.370472:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.370474:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573962.370478:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573962/real 1788573962] req@ffff8fc4ac823100 x1875440516638080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573962.370487:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573962.370488:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573962.370495:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573962.370498:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573962.370500:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573962.370503:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573962.370507:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516638080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573962.370517:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516638080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370525:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573962.370527:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.370529:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573962.370531:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573962.370535:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516638080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370542:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573962.370546:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516638080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370554:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573962.370557:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.370558:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573962.370560:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.370561:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573962.370563:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573962.370566:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573962.370569:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573962.370571:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573962.370575:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516638080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573962.370583:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638080:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573962.370593:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573962.370598:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516638208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370605:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573962.370607:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573962.370610:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573962.370612:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573962.370615:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573962.370618:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638208:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573962.370622:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573962.370624:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573962.370626:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573962.370629:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573962.370632:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573962.370634:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573962.370639:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573962.370641:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573962.370645:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573962.370649:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516638208, portal 10 00000100:00000001:0.0:1788573962.370652:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573962.370653:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788573962.370659:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516638208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370667:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573962.370670:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573962.370672:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573962.370675:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516638208, offset 0 00000400:00000200:0.0:1788573962.370680:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573962.370684:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573962.370688:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573962.370692:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573962.370699:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573962.370702:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573962.370705:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573962.370709:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546638:546638:256:4294967295] 192.168.202.128@tcp LPNI seq info [546561:546561:8:4294967295] 00000400:00000200:0.0:1788573962.370714:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573962.370721:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573962.370727:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573962.370730:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573962.370735:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573962.370738:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573962.370750:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573962.370752:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573962.370756:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573962.370760:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573962.370763:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573962.370765:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573962.370768:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573962.370772:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516638208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370781:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573962.370784:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516638208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370792:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.370793:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573962.370796:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573962.370800:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573962.370804:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573962.370806:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309322266 00000100:00000001:0.0:1788573962.370827:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309326061 00000100:00000001:0.0:1788573962.370830:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309322267 00000100:00000001:0.0:1788573962.370838:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309326061 00000100:00000001:0.0:1788573962.370840:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573962.370843:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.370845:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573962.370851:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573962/real 1788573962] req@ffff8fc4ac821500 x1875440516638208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573962.370861:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573962.370862:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573962.370864:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573962.370867:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573962.370869:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573962.370872:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573962.370876:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516638208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573962.370886:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516638208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370894:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573962.370896:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.370898:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573962.370900:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573962.370904:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516638208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370912:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573962.370916:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516638208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370924:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573962.370927:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.370929:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573962.370930:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.370937:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573962.370940:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573962.370943:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573962.370948:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573962.370950:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573962.370955:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516638208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573962.370965:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638208:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573962.370969:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573962.370973:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516638336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.370981:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573962.370982:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573962.370985:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573962.370987:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573962.370990:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573962.370993:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638336:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573962.370997:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573962.370999:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573962.371001:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573962.371003:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573962.371007:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573962.371009:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573962.371014:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573962.371016:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573962.371020:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573962.371024:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516638336, portal 4 00000100:00000001:0.0:1788573962.371026:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573962.371027:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788573962.371033:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516638336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371047:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573962.371050:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573962.371053:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573962.371056:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516638336, offset 0 00000400:00000200:0.0:1788573962.371061:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573962.371065:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573962.371069:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573962.371072:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573962.371075:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573962.371078:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573962.371081:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573962.371084:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546639:546639:256:4294967295] 192.168.202.128@tcp LPNI seq info [546562:546562:8:4294967295] 00000400:00000200:0.0:1788573962.371089:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573962.371097:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573962.371102:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573962.371105:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573962.371110:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573962.371113:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573962.371126:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573962.371128:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573962.371131:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573962.371135:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573962.371138:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573962.371140:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573962.371143:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573962.371148:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516638336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371156:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573962.371160:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516638336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371179:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.371181:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573962.371184:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573962.371188:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573962.371192:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573962.371194:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309322407 00000100:00000001:0.0:1788573962.371197:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309326061 00000100:00000001:0.0:1788573962.371199:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309322407 00000100:00000001:0.0:1788573962.371202:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309326061 00000100:00000001:0.0:1788573962.371204:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573962.371207:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.371209:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573962.371213:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573962/real 1788573962] req@ffff8fc4ac823b80 x1875440516638336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573962.371222:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573962.371224:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573962.371226:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573962.371228:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573962.371231:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573962.371233:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573962.371237:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516638336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573962.371247:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516638336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371254:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573962.371256:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.371258:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573962.371260:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573962.371264:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516638336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371276:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573962.371281:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516638336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371289:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573962.371292:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.371295:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573962.371296:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.371298:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573962.371300:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573962.371302:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573962.371306:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573962.371308:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573962.371312:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516638336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573962.371320:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638336:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573962.371323:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573962.371326:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516638464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371332:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573962.371334:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573962.371336:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573962.371338:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573962.371340:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573962.371343:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638464:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573962.371347:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573962.371349:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573962.371370:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573962.371372:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573962.371376:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573962.371379:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573962.371383:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573962.371386:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573962.371390:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573962.371393:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516638464, portal 4 00000100:00000001:0.0:1788573962.371395:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573962.371397:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788573962.371404:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516638464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371411:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573962.371414:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573962.371416:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573962.371419:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516638464, offset 0 00000400:00000200:0.0:1788573962.371423:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573962.371428:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573962.371432:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573962.371435:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573962.371438:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573962.371441:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573962.371444:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573962.371447:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546640:546640:256:4294967295] 192.168.202.128@tcp LPNI seq info [546563:546563:8:4294967295] 00000400:00000200:0.0:1788573962.371452:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573962.371459:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573962.371464:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573962.371468:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573962.371472:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573962.371475:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788573962.371488:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573962.371495:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56500. 00000400:00000200:0.0:1788573962.371498:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573962.371503:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573962.371506:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9088 00000400:00000010:0.0:1788573962.371508:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9088. 00000100:00000001:0.0:1788573962.371511:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573962.371515:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516638464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371524:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573962.371528:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516638464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371536:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.371537:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573962.371541:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573962.371544:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573962.371548:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573962.371550:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309322409 00000100:00000001:0.0:1788573962.371553:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309326061 00000100:00000001:0.0:1788573962.371555:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309322409 00000100:00000001:0.0:1788573962.371557:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309326061 00000100:00000001:0.0:1788573962.371560:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573962.371562:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.371564:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573962.371568:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573962/real 1788573962] req@ffff8fc4ac820000 x1875440516638464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573962.371577:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573962.371579:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573962.371581:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573962.371583:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573962.371585:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573962.371593:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573962.371597:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516638464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573962.371607:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516638464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371615:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573962.371617:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.371619:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573962.371621:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573962.371625:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516638464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371633:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573962.371637:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516638464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371646:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573962.371649:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.371651:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573962.371652:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.371654:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573962.371656:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573962.371659:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573962.371663:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573962.371665:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573962.371669:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516638464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573962.371679:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638464:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573962.371683:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.371686:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573962.371694:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516638080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371702:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573962.371704:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573962.371706:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573962.371709:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573962.371711:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573962.371712:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573962.371715:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573962.371717:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788573962.371721:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573962.371722:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573962.371724:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788573962.371727:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573962.371728:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.371731:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573962.371734:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516638208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371742:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573962.371744:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573962.371745:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573962.371748:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573962.371749:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573962.371750:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573962.371753:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573962.371754:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788573962.371757:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573962.371758:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573962.371760:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788573962.371762:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573962.371764:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.371766:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573962.371769:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516638336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371776:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573962.371785:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573962.371787:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573962.371790:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573962.371792:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573962.371793:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573962.371795:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573962.371796:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788573962.371799:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573962.371800:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573962.371802:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788573962.371805:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573962.371806:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.371826:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573962.371830:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516638464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573962.371838:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573962.371840:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573962.371842:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573962.371844:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573962.371846:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573962.371847:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573962.371850:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573962.371851:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788573962.371854:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573962.371855:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573962.371857:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788573962.371859:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573962.371861:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.371863:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.371867:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573962.371869:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.371871:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573962.371872:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573962.880902:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573962.880911:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573962.880917:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573962.880925:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573962.880949:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573962.880956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.880967:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573962.880971:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573962.880975:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573963.585011:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573963.585015:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573963.585019:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573963.904916:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573963.904925:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573963.904927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573963.904932:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573963.904939:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573963.904946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573963.904953:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573963.904970:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573963.904975:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573963.905009:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573963.905012:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573963.905014:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573963.905016:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573963.905027:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573963.905029:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573963.905032:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573963.905033:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573963.905035:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573964.928907:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573964.928912:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573964.928916:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573964.928925:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573964.928934:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573964.928940:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573964.928948:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573964.928953:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573964.928959:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573965.952912:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573965.952920:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573965.952923:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573965.952943:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573965.952948:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573965.952951:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573965.952967:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573965.952969:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573965.952971:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573965.952999:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573965.953004:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573965.953006:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573965.953009:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573965.953013:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573965.953015:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573965.953018:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573965.953020:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573965.953022:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573966.208890:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309326061, at 4309326080 00000100:00000010:0.0:1788573966.208908:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573966.656972:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573966.656976:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573966.656980:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573966.976888:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573966.976894:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573966.976897:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573966.976901:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573966.976906:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573966.976909:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573966.976924:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573966.976928:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573966.976933:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573966.976962:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573966.976967:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573966.976971:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573966.976973:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573966.976978:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573966.976981:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573966.976984:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573966.976986:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573966.977004:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573967.489144:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573967.489148:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573967.489151:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573967.489162:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5180 x1875440516638592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573967.489186:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573967.489191:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573967.489198:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573967.489204:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573967.489209:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573967.489216:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516638592:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573967.489223:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573967.489228:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573967.489232:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573967.489236:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573967.489241:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573967.489245:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573967.489254:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573967.489268:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573967.489278:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573967.489289:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516638592, portal 25 00000100:00000001:0.0:1788573967.489296:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573967.489300:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605325184 : -123400104226432 : ffff8fc4ac9f5180) 00000100:00000040:0.0:1788573967.489314:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5180 x1875440516638592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573967.489332:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573967.489340:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573967.489345:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9088. 00000100:00000200:0.0:1788573967.489351:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516638592, offset 0 00000400:00000200:0.0:1788573967.489369:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573967.489381:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573967.489389:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573967.489404:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573967.489409:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573967.489426:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573967.489432:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573967.489440:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546641:546641:256:4294967295] 192.168.202.128@tcp LPNI seq info [546564:546564:8:4294967295] 00000400:00000200:0.0:1788573967.489450:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573967.489470:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573967.489482:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573967.489490:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788573967.489499:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573967.489532:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573967.489542:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573967.489548:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309331181 00000100:00000010:0.0:1788573967.489558:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573967.489565:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573967.489572:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309331181 00000100:00000010:0.0:1788573967.489580:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b505a0. 00000100:00000001:0.0:1788573967.489586:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573967.489591:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573967.489594:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573967.489596:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573967.489600:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573967.489606:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573967.489610:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573967.489615:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573967.489618:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573967.489620:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573967.489627:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4e00 x1875440516638720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573967.489643:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573967.489646:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573967.489655:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573967.489657:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573967.489662:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573967.489665:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573967.489669:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573967.489673:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573967.489676:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573967.489680:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573967.489685:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573967.489690:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573967.489697:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573967.489702:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788573967.489709:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9a18. 00000100:00000200:0.0:1788573967.489713:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516638720, portal 10 00000100:00000001:0.0:1788573967.489716:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573967.489717:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605324288 : -123400104227328 : ffff8fc4ac9f4e00) 00000100:00000040:0.0:1788573967.489725:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4e00 x1875440516638720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573967.489735:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573967.489740:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573967.489744:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9cc0. 00000100:00000200:0.0:1788573967.489748:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516638720, offset 0 00000400:00000200:0.0:1788573967.489758:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573967.489765:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573967.489772:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573967.489778:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573967.489782:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573967.489786:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573967.489791:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573967.489797:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546642:546642:255:4294967295] 192.168.202.128@tcp LPNI seq info [546565:546565:7:4294967295] 00000400:00000200:0.0:1788573967.489802:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573967.489827:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573967.489837:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573967.489840:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56000. 00000800:00000200:0.0:1788573967.489849:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573967.489855:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573967.489862:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573967.489870:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573967.489877:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309327379 00000100:00000001:0.0:1788573967.489883:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309331181 00000100:00000001:0.0:1788573967.489888:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309327379 00000100:00000001:0.0:1788573967.489894:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309331181 00000100:00000001:0.0:1788573967.489898:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573967.489903:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573967.489908:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573967.489916:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516638848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573967.489934:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573967.489938:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573967.489944:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573967.489948:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573967.489954:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573967.489960:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638848:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573967.489967:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573967.489970:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573967.489974:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573967.489979:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573967.489985:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573967.489990:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573967.489999:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573967.490004:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915790. 00000400:00000010:0.0:1788573967.490011:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573967.490026:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516638848, portal 10 00000100:00000001:0.0:1788573967.490030:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573967.490031:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788573967.490041:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516638848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573967.490048:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573967.490050:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573967.490052:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9dd0. 00000100:00000200:0.0:1788573967.490055:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516638848, offset 0 00000400:00000200:0.0:1788573967.490059:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573967.490064:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573967.490068:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573967.490071:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573967.490073:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573967.490075:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573967.490077:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573967.490086:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546643:546643:254:4294967295] 192.168.202.128@tcp LPNI seq info [546566:546566:6:4294967295] 00000400:00000200:0.0:1788573967.490091:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573967.490097:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573967.490102:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573967.490105:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788573967.490110:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573967.490116:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573967.490119:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573967.490122:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573967.490124:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309327386 00000100:00000001:0.0:1788573967.490127:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309331181 00000100:00000001:0.0:1788573967.490129:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309327386 00000100:00000001:0.0:1788573967.490130:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309331181 00000100:00000001:0.0:1788573967.490132:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573967.490141:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573967.490143:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573967.490145:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573967.490148:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573967.490150:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573967.490152:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573967.490153:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573967.490155:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573967.490159:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6680 x1875440516638976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573967.490167:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573967.490168:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573967.490170:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573967.490173:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573967.490175:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573967.490178:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573967.490181:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573967.490183:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573967.490185:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573967.490186:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573967.490190:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573967.490192:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573967.490196:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573967.490199:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788573967.490203:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9908. 00000100:00000200:0.0:1788573967.490206:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516638976, portal 4 00000100:00000001:0.0:1788573967.490209:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573967.490210:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605330560 : -123400104221056 : ffff8fc4ac9f6680) 00000100:00000040:0.0:1788573967.490216:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6680 x1875440516638976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573967.490222:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573967.490224:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573967.490226:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9aa0. 00000100:00000200:0.0:1788573967.490228:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516638976, offset 0 00000400:00000200:0.0:1788573967.490239:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573967.490243:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573967.490247:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573967.490250:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573967.490252:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573967.490255:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573967.490257:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573967.490260:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546644:546644:253:4294967295] 192.168.202.128@tcp LPNI seq info [546567:546567:5:4294967295] 00000400:00000200:0.0:1788573967.490264:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573967.490269:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573967.490275:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573967.490279:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788573967.490282:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573967.490288:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573967.490291:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573967.490295:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573967.490297:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309327526 00000100:00000001:0.0:1788573967.490299:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309331181 00000100:00000001:0.0:1788573967.490301:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309327526 00000100:00000001:0.0:1788573967.490303:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309331181 00000100:00000001:0.0:1788573967.490306:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573967.490308:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573967.490310:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573967.490313:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7480 x1875440516639104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573967.490320:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573967.490321:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573967.490323:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573967.490325:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573967.490327:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573967.490336:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516639104:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573967.490340:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573967.490342:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573967.490344:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573967.490345:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573967.490349:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573967.490351:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573967.490354:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573967.490357:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915948. 00000400:00000010:0.0:1788573967.490360:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9d48. 00000100:00000200:0.0:1788573967.490363:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516639104, portal 4 00000100:00000001:0.0:1788573967.490365:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573967.490366:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605334144 : -123400104217472 : ffff8fc4ac9f7480) 00000100:00000040:0.0:1788573967.490373:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7480 x1875440516639104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788573992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573967.490379:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573967.490381:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573967.490383:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9e58. 00000100:00000200:0.0:1788573967.490386:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516639104, offset 0 00000400:00000200:0.0:1788573967.490390:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573967.490394:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573967.490397:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573967.490400:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573967.490403:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573967.490405:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573967.490408:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573967.490411:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546645:546645:252:4294967295] 192.168.202.128@tcp LPNI seq info [546568:546568:4:4294967295] 00000400:00000200:0.0:1788573967.490429:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573967.490436:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573967.490441:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573967.490445:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56f00. 00000800:00000200:0.0:1788573967.490455:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788573967.490462:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573967.490465:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573967.490469:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573967.490472:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309327528 00000100:00000001:0.0:1788573967.490474:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309331181 00000100:00000001:0.0:1788573967.490476:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309327528 00000100:00000001:0.0:1788573967.490478:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309331181 00000100:00000001:0.0:1788573967.490481:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573967.490483:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573967.490487:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573967.490489:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573967.490493:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573967.490494:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788573967.490496:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573967.490498:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573967.490499:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573967.490502:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573968.000929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573968.000940:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573968.000944:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573968.000966:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573968.000972:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573968.000976:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573968.001005:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573968.001007:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573968.001010:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573968.705022:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573968.705026:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573968.705029:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573969.024899:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573969.024908:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573969.024913:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573969.024923:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573969.024931:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573969.024955:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573969.024969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573969.024972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573969.024976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573970.048923:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573970.048932:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573970.048936:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573970.048943:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573970.048955:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573970.048962:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.048986:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573970.048990:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.048995:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573970.561601:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573970.561606:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573970.561608:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573970.561616:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573967/real 1788573970] req@ffff8fc4ac9f6680 x1875440516638976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573970.561628:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573970.561630:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573970.561634:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19152c0. 00000400:00000200:0.0:1788573970.561638:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9908 00000400:00000010:0.0:1788573970.561640:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9908. 00000100:00000001:0.0:1788573970.561644:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573970.561653:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6680 x1875440516638976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573970.561663:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6680 x1875440516638976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.561674:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573970.561676:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.561679:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573970.561681:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573970.561686:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6680 x1875440516638976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.561706:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573970.561711:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6680 x1875440516638976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.561719:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573970.561722:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.561724:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573970.561725:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.561728:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573970.561731:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573970.561735:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573970.561740:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573970.561743:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573970.561747:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6680 x1875440516638976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573970.561754:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573970.561759:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573970.561762:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573967/real 1788573970] req@ffff8fc4ac9f7480 x1875440516639104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573970.561773:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573970.561774:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573970.561776:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915948. 00000400:00000200:0.0:1788573970.561779:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9d48 00000400:00000010:0.0:1788573970.561781:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9d48. 00000100:00000001:0.0:1788573970.561783:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573970.561787:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7480 x1875440516639104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573970.561848:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7480 x1875440516639104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.561856:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573970.561858:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.561860:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573970.561862:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573970.561866:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7480 x1875440516639104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.561873:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573970.561878:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7480 x1875440516639104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.561885:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573970.561887:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.561889:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573970.561891:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.561892:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573970.561895:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573970.561897:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573970.561901:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573970.561904:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573970.561908:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7480 x1875440516639104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573970.561918:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516639104:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573970.561922:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573970.561926:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573967/real 1788573970] req@ffff8fc4ac9f4e00 x1875440516638720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573970.561974:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573970.561976:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573970.561979:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b58. 00000400:00000200:0.0:1788573970.561982:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9a18 00000400:00000010:0.0:1788573970.561984:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9a18. 00000100:00000001:0.0:1788573970.561986:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573970.561990:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4e00 x1875440516638720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573970.561999:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4e00 x1875440516638720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562007:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573970.562008:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562010:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573970.562012:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573970.562016:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4e00 x1875440516638720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562022:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573970.562025:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4e00 x1875440516638720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562033:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573970.562036:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562038:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573970.562039:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562041:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573970.562043:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573970.562046:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573970.562050:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573970.562054:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573970.562066:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4e00 x1875440516638720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573970.562074:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573970.562078:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573970.562082:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573967/real 1788573970] req@ffff8fc4ac9f6a00 x1875440516638848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573970.562090:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573970.562092:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573970.562094:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915790. 00000400:00000200:0.0:1788573970.562096:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573970.562098:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573970.562101:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573970.562104:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516638848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573970.562114:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516638848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562121:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573970.562122:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562124:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573970.562126:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573970.562129:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516638848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562136:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573970.562140:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516638848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562147:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573970.562149:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562156:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573970.562158:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562160:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573970.562163:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573970.562166:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573970.562170:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573970.562172:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573970.562176:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516638848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573970.562184:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516638848:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573970.562189:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573970.562192:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573967/real 1788573970] req@ffff8fc4ac9f5180 x1875440516638592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573970.562199:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573970.562200:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573970.562202:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573970.562205:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573970.562206:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573970.562209:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573970.562212:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516638592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573970.562220:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516638592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562226:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573970.562227:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562229:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573970.562230:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573970.562234:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516638592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562250:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573970.562255:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516638592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562262:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573970.562264:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562266:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573970.562267:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562268:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573970.562270:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573970.562273:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573970.562276:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573970.562278:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573970.562282:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516638592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573970.562290:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516638592:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573970.562295:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562297:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573970.562301:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6680 x1875440516638976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562308:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573970.562311:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573970.562315:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573970.562320:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573970.562323:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573970.562326:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573970.562329:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573970.562332:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788573970.562337:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573970.562345:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573970.562348:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6680. 00000100:00000001:0.0:1788573970.562353:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573970.562354:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562356:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573970.562360:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7480 x1875440516639104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562365:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573970.562366:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573970.562368:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573970.562371:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573970.562372:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573970.562374:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573970.562376:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573970.562378:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788573970.562380:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573970.562381:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573970.562383:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7480. 00000100:00000001:0.0:1788573970.562385:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573970.562386:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562388:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573970.562391:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4e00 x1875440516638720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562398:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573970.562399:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573970.562401:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573970.562404:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573970.562405:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573970.562406:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573970.562408:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573970.562409:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34400. 02000000:00000001:0.0:1788573970.562412:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573970.562413:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573970.562414:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4e00. 00000100:00000001:0.0:1788573970.562417:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573970.562418:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562425:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573970.562429:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516638848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562435:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573970.562437:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573970.562439:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573970.562441:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573970.562443:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573970.562444:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573970.562446:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573970.562448:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788573970.562464:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573970.562466:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573970.562467:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788573970.562470:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573970.562472:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562474:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573970.562477:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516638592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788573992 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573970.562484:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573970.562485:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573970.562487:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788573970.562490:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573970.562491:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573970.562492:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573970.562494:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573970.562496:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36c00. 02000000:00000001:0.0:1788573970.562498:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573970.562500:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573970.562501:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788573970.562504:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573970.562505:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562507:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573970.562517:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573970.562519:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573970.562522:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573970.562532:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573971.072888:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573971.072896:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573971.072898:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573971.072907:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573971.072913:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573971.072918:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573971.072924:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573971.072926:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573971.072931:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573971.072962:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573971.072966:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573971.072969:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573971.072975:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573971.072978:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573971.072981:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573971.072987:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573971.072990:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573971.072993:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573971.328925:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309331181, at 4309331200 00000100:00000010:0.0:1788573971.328938:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573971.776989:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573971.776992:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573971.776995:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573972.096959:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573972.096969:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573972.096974:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573972.096984:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573972.096997:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573972.097003:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573972.097033:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573972.097039:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573972.097045:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.097065:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573972.097068:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573972.097070:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573972.097073:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573972.097076:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573972.097099:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573972.097102:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573972.097104:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573972.097108:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573972.608971:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573972.608983:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573972.608987:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573972.608990:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573972.608995:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573972.609001:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14660 00000400:00000200:0.0:1788573972.609010:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573972.609015:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573972.609018:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573972.609020:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609023:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573972.609024:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609027:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573972.609030:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573972.609031:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573972.609033:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573972.609036:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573972.609037:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573972.609039:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573972.609041:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609043:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573972.609044:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609045:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573972.609047:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573972.609049:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609051:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573972.609053:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573972.609073:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573972.609076:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.609077:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573972.609079:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573972.609086:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788573972.609089:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573972.609092:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573972.609093:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.609095:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573972.609097:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573972.609101:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573972.609107:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788573972.609113:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573972.609120:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516639232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573972.609129:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573972.609133:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573972.609136:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516639232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573972.609162:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573972.609166:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573972.609169:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573972.609171:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573972.609172:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573972.609175:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573972.609178:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14660 00000400:00000200:0.0:1788573972.609182:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573972.609186:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573972.609189:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573972.609192:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609195:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573972.609204:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609206:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573972.609209:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573972.609213:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573972.609216:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573972.609222:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573972.609226:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573972.609229:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573972.609235:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609240:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573972.609245:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609250:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573972.609255:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573972.609262:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609267:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573972.609271:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573972.609279:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573972.609281:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.609289:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573972.609301:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573972.609308:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.609311:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573972.609318:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788573972.609324:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573972.609328:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573972.609333:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.609337:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573972.609352:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573972.609356:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573972.609364:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788573972.609371:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573972.609381:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516639360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573972.609416:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573972.609422:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573972.609431:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516639360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573972.609448:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573972.609454:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573972.609461:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573972.609464:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573972.609468:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573972.609491:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573972.609499:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14660 00000400:00000200:0.0:1788573972.609511:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573972.609520:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573972.609525:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573972.609531:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609534:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573972.609540:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609547:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573972.609559:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573972.609561:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573972.609563:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573972.609566:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573972.609571:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573972.609574:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573972.609580:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609584:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573972.609590:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609594:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573972.609607:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573972.609627:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609634:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573972.609637:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573972.609642:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573972.609648:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.609654:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573972.609658:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573972.609662:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.609667:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573972.609672:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788573972.609678:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573972.609681:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573972.609685:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.609688:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573972.609691:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573972.609695:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573972.609702:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788573972.609708:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573972.609715:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516639488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573972.609722:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573972.609725:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573972.609728:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516639488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573972.609735:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573972.609738:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573972.609741:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573972.609743:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573972.609744:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573972.609747:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573972.609756:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14660 00000400:00000200:0.0:1788573972.609761:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573972.609764:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573972.609767:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573972.609770:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609772:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573972.609774:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609776:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573972.609778:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573972.609779:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573972.609780:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573972.609782:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573972.609784:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573972.609786:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573972.609789:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609791:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573972.609792:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609794:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573972.609796:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573972.609799:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609801:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573972.609803:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573972.609806:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573972.609870:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.609873:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573972.609876:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573972.609878:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.609880:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573972.609884:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788573972.609887:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573972.609891:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573972.609900:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.609902:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573972.609904:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573972.609906:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573972.609911:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788573972.609914:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573972.609920:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516639616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573972.609928:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573972.609930:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573972.609934:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516639616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573972.609941:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573972.609944:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573972.609947:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573972.609949:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573972.609951:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573972.609953:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573972.609956:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14660 00000400:00000200:0.0:1788573972.609960:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573972.609964:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573972.609966:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573972.609969:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609971:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573972.609973:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609975:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573972.609978:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573972.609979:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573972.609980:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573972.609982:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573972.609984:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573972.609992:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573972.609996:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.609999:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573972.610001:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573972.610003:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573972.610006:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573972.610008:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573972.610011:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573972.610013:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573972.610016:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573972.610018:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.610020:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573972.610022:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573972.610024:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.610026:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573972.610029:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788573972.610032:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573972.610035:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573972.610036:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573972.610038:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573972.610039:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573972.610041:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573972.610045:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788573972.610049:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573972.610054:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516639744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573972.610061:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573972.610063:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573972.610067:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516639744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573972.610074:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573972.610085:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14670) 00000100:00000001:0.0:1788573973.120924:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573973.120931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573973.120933:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573973.120941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573973.120948:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573973.120963:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573973.120982:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573973.120991:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573973.120997:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573973.825082:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573973.825090:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573973.825095:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573974.145005:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573974.145015:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573974.145021:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573974.145033:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573974.145048:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573974.145053:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573974.145068:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573974.145077:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573974.145081:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573975.169016:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573975.169022:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573975.169029:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573975.169041:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788573975.169049:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573975.169065:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573975.169099:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573975.169104:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573975.169108:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573976.192941:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573976.192949:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573976.192955:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573976.192966:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573976.192978:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573976.192985:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573976.193016:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573976.193038:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573976.193048:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573976.448930:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309336302, at 4309336320 00000100:00000010:0.0:1788573976.448943:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7540. 00000020:00000001:0.0:1788573976.897102:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573976.897108:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573976.897113:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573977.216955:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573977.216967:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573977.216973:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573977.216982:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573977.216992:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573977.217003:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573977.217042:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573977.217047:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573977.217054:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573977.728933:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573977.728945:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788573977.728948:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573977.728953:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573977.728959:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573977.728968:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14665 00000400:00000200:0.0:1788573977.728978:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573977.728986:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573977.728990:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573977.728994:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.728997:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573977.729000:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729002:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573977.729007:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573977.729009:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788573977.729012:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573977.729015:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788573977.729036:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788573977.729039:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788573977.729044:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729046:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573977.729048:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729050:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573977.729054:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788573977.729057:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729060:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573977.729063:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573977.729066:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573977.729069:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.729072:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788573977.729074:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573977.729080:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788573977.729084:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788573977.729088:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573977.729090:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.729093:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573977.729096:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573977.729100:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573977.729107:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573977.729121:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573977.729129:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516639872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573977.729141:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573977.729145:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573977.729150:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516639872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573977.729177:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573977.729180:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573977.729192:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788573977.729194:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573977.729197:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573977.729200:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573977.729204:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14665 00000400:00000200:0.0:1788573977.729210:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573977.729227:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573977.729233:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573977.729239:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729245:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573977.729252:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729263:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573977.729279:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573977.729284:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788573977.729290:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573977.729296:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788573977.729306:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788573977.729313:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788573977.729325:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729331:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573977.729337:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729344:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573977.729352:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788573977.729360:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729364:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573977.729373:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573977.729383:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573977.729392:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.729398:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573977.729407:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573977.729423:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.729428:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573977.729433:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788573977.729438:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573977.729441:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573977.729444:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.729448:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573977.729453:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573977.729457:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573977.729466:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788573977.729476:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573977.729492:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516640000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573977.729531:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573977.729536:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573977.729547:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516640000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573977.729578:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573977.729582:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573977.729586:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788573977.729592:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573977.729599:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573977.729607:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573977.729617:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14665 00000400:00000200:0.0:1788573977.729631:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573977.729655:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573977.729662:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573977.729672:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729684:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573977.729696:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729714:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573977.729724:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573977.729730:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788573977.729736:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573977.729742:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788573977.729749:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788573977.729757:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788573977.729768:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729774:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573977.729783:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729790:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573977.729798:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788573977.729806:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573977.729847:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573977.729849:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573977.729857:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573977.729861:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.729865:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573977.729872:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788573977.729880:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.729883:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573977.729894:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788573977.729903:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788573977.729914:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573977.729917:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.729920:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573977.729923:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573977.729931:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573977.729942:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788573977.729949:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573977.729963:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516640128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573977.730023:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573977.730032:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573977.730043:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516640128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573977.730060:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573977.730067:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573977.730078:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788573977.730084:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573977.730087:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573977.730097:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573977.730105:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14665 00000400:00000200:0.0:1788573977.730116:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573977.730125:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573977.730128:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573977.730132:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.730138:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573977.730144:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.730155:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573977.730162:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573977.730167:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788573977.730173:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573977.730178:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788573977.730185:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788573977.730198:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788573977.730209:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.730218:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573977.730228:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.730236:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573977.730253:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788573977.730262:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573977.730265:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573977.730267:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573977.730274:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573977.730282:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.730289:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573977.730295:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573977.730301:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.730310:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573977.730322:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788573977.730330:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573977.730345:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573977.730349:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.730354:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573977.730362:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573977.730368:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573977.730379:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788573977.730388:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573977.730403:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516640256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573977.730431:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573977.730438:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573977.730453:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516640256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573977.730491:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788573977.730499:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788573977.730519:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788573977.730525:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788573977.730530:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788573977.730539:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788573977.730555:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14665 00000400:00000200:0.0:1788573977.730567:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573977.730574:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788573977.730582:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788573977.730592:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.730598:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573977.730607:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.730613:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788573977.730621:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788573977.730625:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788573977.730632:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788573977.730639:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788573977.730645:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788573977.730651:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788573977.730654:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.730657:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788573977.730665:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788573977.730673:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788573977.730679:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788573977.730688:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788573977.730696:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573977.730704:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788573977.730711:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573977.730719:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.730726:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788573977.730731:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788573977.730739:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.730745:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788573977.730755:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788573977.730766:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788573977.730774:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788573977.730788:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573977.730794:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788573977.730800:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788573977.730805:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788573977.730848:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788573977.730859:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573977.730874:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516640384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788573977.730894:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788573977.730899:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573977.730908:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516640384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573977.730943:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573977.730956:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14675) 00000100:00000001:0.0:1788573978.240988:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573978.241000:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573978.241007:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573978.241018:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573978.241035:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573978.241046:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573978.241061:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573978.241066:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573978.241069:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573978.945032:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573978.945037:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573978.945040:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573980.288991:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573980.288997:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573980.289000:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573980.289012:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573980.289015:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573980.289019:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573980.289023:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573980.289025:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573980.289030:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573981.312921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573981.312926:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573981.312930:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573981.312934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573981.312938:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573981.312941:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573981.312954:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573981.312959:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573981.312963:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573981.568905:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309341422, at 4309341440 00000100:00000010:0.0:1788573981.568921:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea77e0. 00000020:00000001:0.0:1788573982.016970:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573982.016973:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573982.016975:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573982.336944:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573982.336951:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573982.336955:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573982.336960:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573982.336966:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573982.336980:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.336987:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573982.336990:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.336995:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.849134:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573982.849141:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573982.849146:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573982.849159:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516640512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.849184:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573982.849188:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573982.849193:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573982.849198:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573982.849204:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573982.849215:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516640512:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573982.849224:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573982.849238:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573982.849243:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573982.849248:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573982.849255:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573982.849260:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573982.849273:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573982.849280:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573982.849290:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573982.849299:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516640512, portal 25 00000100:00000001:0.0:1788573982.849305:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573982.849309:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788573982.849320:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516640512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.849338:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573982.849347:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573982.849352:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573982.849369:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516640512, offset 0 00000400:00000200:0.0:1788573982.849382:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573982.849393:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573982.849403:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573982.849411:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573982.849420:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573982.849428:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573982.849435:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573982.849444:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546656:546656:256:4294967295] 192.168.202.128@tcp LPNI seq info [546579:546579:8:4294967295] 00000400:00000200:0.0:1788573982.849456:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573982.849476:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573982.849488:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573982.849497:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573982.849509:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573982.849517:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788573982.849567:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573982.849582:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573982.849591:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573982.849606:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573982.849616:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573982.849621:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573982.849629:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573982.849638:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516640512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.849653:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573982.849661:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516640512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.849678:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.849683:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573982.849691:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573982.849701:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573982.849707:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309346541 00000100:00000010:0.0:1788573982.849716:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573982.849726:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573982.849735:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309346541 00000100:00000010:0.0:1788573982.849742:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50f00. 00000100:00000001:0.0:1788573982.849749:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573982.849757:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573982.849763:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.849767:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573982.849776:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573982/real 1788573982] req@ffff8fc4901a4a80 x1875440516640512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573982.849800:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573982.849803:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573982.849827:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573982.849833:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573982.849837:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573982.849844:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573982.849858:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516640512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573982.849879:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516640512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.849895:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573982.849900:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.849907:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573982.849911:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573982.849921:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516640512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.849936:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573982.849944:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516640512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.849952:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573982.849956:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.849959:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573982.849961:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.849964:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573982.849968:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573982.849972:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573982.849977:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573982.849981:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573982.849987:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516640512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573982.849997:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516640512:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573982.850003:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.850005:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573982.850008:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516640512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850027:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573982.850029:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573982.850032:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573982.850036:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573982.850038:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573982.850039:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573982.850041:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573982.850043:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788573982.850047:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573982.850049:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573982.850051:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788573982.850056:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573982.850057:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.850060:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.850066:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573982.850069:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.850071:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573982.850073:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573982.850074:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573982.850079:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516640640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850087:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573982.850089:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573982.850091:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573982.850093:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573982.850095:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573982.850098:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516640640:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573982.850102:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573982.850104:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573982.850106:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573982.850108:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573982.850111:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573982.850113:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573982.850117:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573982.850125:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573982.850129:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573982.850132:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516640640, portal 10 00000100:00000001:0.0:1788573982.850135:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573982.850137:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788573982.850144:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516640640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850152:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573982.850154:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573982.850157:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573982.850159:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516640640, offset 0 00000400:00000200:0.0:1788573982.850163:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573982.850167:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573982.850171:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573982.850175:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573982.850177:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573982.850180:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573982.850182:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573982.850186:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546657:546657:256:4294967295] 192.168.202.128@tcp LPNI seq info [546580:546580:8:4294967295] 00000400:00000200:0.0:1788573982.850190:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573982.850197:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573982.850202:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573982.850205:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573982.850209:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573982.850212:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788573982.850224:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573982.850226:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573982.850229:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573982.850233:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573982.850236:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573982.850238:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573982.850246:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573982.850251:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516640640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850260:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573982.850263:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516640640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850270:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.850272:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573982.850275:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573982.850279:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573982.850283:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573982.850285:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309342739 00000100:00000001:0.0:1788573982.850288:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309346541 00000100:00000001:0.0:1788573982.850291:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309342739 00000100:00000001:0.0:1788573982.850293:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309346541 00000100:00000001:0.0:1788573982.850295:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573982.850297:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.850300:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573982.850303:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573982/real 1788573982] req@ffff8fc4901a7800 x1875440516640640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573982.850312:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573982.850313:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573982.850315:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573982.850318:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573982.850320:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573982.850322:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573982.850326:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516640640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573982.850334:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516640640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850352:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573982.850354:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.850356:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573982.850357:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573982.850362:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516640640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850369:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573982.850373:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516640640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850380:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573982.850382:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.850384:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573982.850385:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.850387:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573982.850389:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573982.850392:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573982.850396:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573982.850398:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573982.850402:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516640640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573982.850410:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516640640:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573982.850414:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573982.850417:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516640768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850424:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573982.850425:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573982.850432:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573982.850434:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573982.850436:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573982.850440:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516640768:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573982.850443:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573982.850445:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573982.850447:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573982.850449:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573982.850452:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573982.850454:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573982.850458:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573982.850461:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573982.850464:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573982.850467:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516640768, portal 10 00000100:00000001:0.0:1788573982.850469:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573982.850471:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788573982.850477:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516640768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850484:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573982.850486:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573982.850488:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573982.850490:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516640768, offset 0 00000400:00000200:0.0:1788573982.850494:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573982.850498:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573982.850502:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573982.850505:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573982.850508:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573982.850511:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573982.850513:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573982.850517:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546658:546658:256:4294967295] 192.168.202.128@tcp LPNI seq info [546581:546581:8:4294967295] 00000400:00000200:0.0:1788573982.850535:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573982.850543:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573982.850556:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573982.850560:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573982.850565:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573982.850568:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788573982.850579:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573982.850581:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573982.850584:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573982.850588:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573982.850591:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573982.850593:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573982.850596:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573982.850600:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516640768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850607:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573982.850610:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516640768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850617:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.850619:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573982.850622:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573982.850626:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573982.850629:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573982.850631:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309342746 00000100:00000001:0.0:1788573982.850634:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309346541 00000100:00000001:0.0:1788573982.850636:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309342746 00000100:00000001:0.0:1788573982.850638:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309346541 00000100:00000001:0.0:1788573982.850641:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573982.850643:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.850644:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573982.850648:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573982/real 1788573982] req@ffff8fc4901a5180 x1875440516640768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573982.850661:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573982.850663:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573982.850665:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573982.850668:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573982.850669:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573982.850672:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573982.850675:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516640768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573982.850685:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516640768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850692:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573982.850693:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.850695:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573982.850697:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573982.850700:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516640768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850708:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573982.850712:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516640768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850719:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573982.850722:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.850723:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573982.850725:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.850727:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573982.850729:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573982.850732:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573982.850736:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573982.850738:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573982.850748:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516640768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573982.850757:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516640768:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573982.850761:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573982.850765:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516640896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850771:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573982.850773:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573982.850775:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573982.850777:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573982.850779:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573982.850782:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516640896:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573982.850785:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573982.850787:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573982.850788:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573982.850790:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573982.850794:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573982.850796:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573982.850800:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573982.850803:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573982.850806:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573982.850828:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516640896, portal 4 00000100:00000001:0.0:1788573982.850830:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573982.850831:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788573982.850837:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516640896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850844:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573982.850847:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573982.850849:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573982.850852:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516640896, offset 0 00000400:00000200:0.0:1788573982.850862:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573982.850866:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573982.850869:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573982.850872:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573982.850876:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573982.850878:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573982.850881:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573982.850884:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546659:546659:256:4294967295] 192.168.202.128@tcp LPNI seq info [546582:546582:8:4294967295] 00000400:00000200:0.0:1788573982.850888:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573982.850895:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573982.850899:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573982.850903:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573982.850907:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573982.850909:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788573982.850920:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573982.850922:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573982.850925:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573982.850929:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573982.850933:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573982.850934:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573982.850937:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573982.850941:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516640896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850948:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573982.850951:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516640896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.850958:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.850960:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573982.850963:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573982.850966:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573982.850969:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573982.850976:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309342887 00000100:00000001:0.0:1788573982.850979:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309346541 00000100:00000001:0.0:1788573982.850981:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309342887 00000100:00000001:0.0:1788573982.850984:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309346541 00000100:00000001:0.0:1788573982.850986:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573982.850988:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.850990:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573982.850995:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573982/real 1788573982] req@ffff8fc4901a5c00 x1875440516640896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573982.851003:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573982.851004:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573982.851006:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573982.851009:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573982.851010:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573982.851013:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573982.851017:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516640896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573982.851025:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516640896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851032:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573982.851033:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.851035:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573982.851037:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573982.851040:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516640896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851047:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573982.851052:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516640896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851059:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573982.851066:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.851068:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573982.851069:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.851071:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573982.851073:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573982.851076:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573982.851080:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573982.851082:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573982.851086:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516640896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573982.851095:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516640896:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573982.851099:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573982.851103:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516641024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851108:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573982.851109:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573982.851111:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573982.851112:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573982.851114:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573982.851115:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516641024:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573982.851117:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573982.851119:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573982.851120:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573982.851121:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573982.851124:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573982.851125:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573982.851127:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573982.851129:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573982.851131:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573982.851134:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516641024, portal 4 00000100:00000001:0.0:1788573982.851141:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573982.851143:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788573982.851149:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516641024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851155:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573982.851158:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573982.851161:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573982.851164:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516641024, offset 0 00000400:00000200:0.0:1788573982.851168:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573982.851172:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573982.851176:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573982.851179:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573982.851182:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573982.851184:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573982.851187:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573982.851190:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546660:546660:256:4294967295] 192.168.202.128@tcp LPNI seq info [546583:546583:8:4294967295] 00000400:00000200:0.0:1788573982.851194:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573982.851199:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573982.851202:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573982.851204:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573982.851207:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573982.851209:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788573982.851216:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573982.851218:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573982.851220:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573982.851222:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573982.851224:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573982.851225:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573982.851227:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573982.851230:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516641024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851244:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573982.851247:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516641024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851251:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.851252:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573982.851255:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573982.851257:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573982.851259:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573982.851261:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309342889 00000100:00000001:0.0:1788573982.851262:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309346541 00000100:00000001:0.0:1788573982.851264:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309342889 00000100:00000001:0.0:1788573982.851265:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309346541 00000100:00000001:0.0:1788573982.851267:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573982.851268:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.851269:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573982.851271:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573982/real 1788573982] req@ffff8fc4901a7100 x1875440516641024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573982.851276:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573982.851277:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573982.851278:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573982.851280:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573982.851281:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573982.851284:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573982.851287:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516641024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573982.851295:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516641024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851302:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573982.851309:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.851311:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573982.851313:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573982.851317:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516641024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851325:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573982.851329:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516641024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851335:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573982.851337:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.851338:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573982.851339:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.851340:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573982.851341:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573982.851343:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573982.851345:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573982.851347:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573982.851349:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516641024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573982.851355:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516641024:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573982.851357:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.851358:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573982.851360:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516640640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851364:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573982.851365:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573982.851367:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573982.851369:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573982.851370:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573982.851375:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573982.851376:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573982.851378:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788573982.851380:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573982.851381:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573982.851382:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788573982.851385:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573982.851386:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.851387:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573982.851389:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516640768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851394:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573982.851395:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573982.851396:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573982.851397:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573982.851397:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573982.851398:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573982.851399:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573982.851400:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788573982.851401:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573982.851402:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573982.851403:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788573982.851406:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573982.851407:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.851408:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573982.851409:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516640896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851413:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573982.851414:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573982.851415:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573982.851416:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573982.851417:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573982.851417:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573982.851418:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573982.851419:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7400. 02000000:00000001:0.0:1788573982.851421:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573982.851424:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573982.851425:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788573982.851427:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573982.851428:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.851429:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573982.851431:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516641024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573982.851435:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573982.851435:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573982.851436:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573982.851438:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573982.851438:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573982.851439:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573982.851440:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573982.851441:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7000. 02000000:00000001:0.0:1788573982.851442:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573982.851443:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573982.851443:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788573982.851445:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573982.851445:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.851446:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573982.851449:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573982.851450:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573982.851451:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573982.851452:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573983.360938:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573983.360945:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573983.360948:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573983.360954:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573983.360959:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573983.360964:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573983.360970:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573983.360974:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573983.360978:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573984.064980:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573984.064983:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573984.064986:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573984.384917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573984.384924:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573984.384929:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573984.384938:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573984.384946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573984.384949:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573984.384953:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573984.384956:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573984.384960:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573985.408992:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573985.408999:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573985.409004:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573985.409012:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573985.409022:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573985.409032:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573985.409042:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573985.409047:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573985.409062:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573986.432970:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573986.432983:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573986.432988:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573986.432998:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573986.433008:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573986.433016:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573986.433032:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573986.433036:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573986.433039:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573986.688959:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309346541, at 4309346560 00000100:00000010:0.0:1788573986.688976:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573987.137035:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573987.137041:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573987.137045:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573987.456990:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573987.456997:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573987.457013:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573987.457024:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573987.457043:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573987.457059:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.457074:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573987.457077:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.457080:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.969262:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573987.969270:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573987.969277:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573987.969289:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516641152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.969304:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573987.969310:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573987.969318:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573987.969324:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573987.969331:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573987.969339:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516641152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573987.969348:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573987.969356:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573987.969360:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573987.969365:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573987.969374:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573987.969379:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573987.969389:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573987.969396:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573987.969410:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573987.969423:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516641152, portal 25 00000100:00000001:0.0:1788573987.969431:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573987.969436:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788573987.969451:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516641152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.969469:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573987.969477:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573987.969484:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573987.969488:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516641152, offset 0 00000400:00000200:0.0:1788573987.969502:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573987.969523:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573987.969532:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573987.969552:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573987.969560:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573987.969566:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573987.969573:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573987.969580:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546661:546661:256:4294967295] 192.168.202.128@tcp LPNI seq info [546584:546584:8:4294967295] 00000400:00000200:0.0:1788573987.969588:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573987.969605:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573987.969613:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573987.969621:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573987.969634:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573987.969640:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788573987.969673:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573987.969676:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573987.969683:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573987.969690:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573987.969696:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573987.969700:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573987.969706:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573987.969712:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516641152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.969725:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573987.969729:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516641152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.969735:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.969737:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573987.969744:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573987.969752:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573987.969755:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309351661 00000100:00000010:0.0:1788573987.969761:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573987.969774:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573987.969783:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309351661 00000100:00000010:0.0:1788573987.969788:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50720. 00000100:00000001:0.0:1788573987.969791:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573987.969797:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573987.969802:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.969829:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573987.969836:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573987/real 1788573987] req@ffff8fc4901a5880 x1875440516641152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573987.969853:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573987.969856:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573987.969860:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573987.969866:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573987.969869:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573987.969873:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573987.969880:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516641152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573987.969899:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516641152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.969913:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573987.969916:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.969920:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573987.969924:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573987.969932:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516641152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.969947:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573987.969952:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516641152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.969961:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573987.969967:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.969977:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573987.969980:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.969983:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573987.969989:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573987.969995:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573987.970000:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573987.970003:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573987.970010:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516641152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573987.970022:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516641152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573987.970031:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.970034:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573987.970038:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516641152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970052:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573987.970055:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573987.970059:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573987.970063:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573987.970066:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573987.970069:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573987.970074:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573987.970076:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6400. 02000000:00000001:0.0:1788573987.970081:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573987.970084:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573987.970087:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788573987.970094:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573987.970097:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.970101:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.970110:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573987.970114:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.970118:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573987.970128:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573987.970131:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573987.970143:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516641280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970153:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573987.970155:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573987.970159:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573987.970161:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573987.970164:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573987.970169:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516641280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573987.970173:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573987.970175:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573987.970178:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573987.970180:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573987.970184:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573987.970187:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573987.970192:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573987.970195:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573987.970199:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573987.970203:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516641280, portal 10 00000100:00000001:0.0:1788573987.970205:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573987.970206:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788573987.970214:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516641280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970221:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573987.970223:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573987.970226:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573987.970229:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516641280, offset 0 00000400:00000200:0.0:1788573987.970233:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573987.970237:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573987.970241:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573987.970244:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573987.970247:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573987.970250:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573987.970258:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573987.970261:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546662:546662:256:4294967295] 192.168.202.128@tcp LPNI seq info [546585:546585:8:4294967295] 00000400:00000200:0.0:1788573987.970266:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573987.970273:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573987.970279:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573987.970283:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573987.970287:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573987.970290:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788573987.970303:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573987.970305:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573987.970307:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573987.970310:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573987.970312:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573987.970314:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573987.970316:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573987.970319:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516641280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970325:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573987.970327:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516641280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970332:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.970333:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573987.970336:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573987.970339:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573987.970341:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573987.970343:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309347859 00000100:00000001:0.0:1788573987.970345:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309351661 00000100:00000001:0.0:1788573987.970347:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309347859 00000100:00000001:0.0:1788573987.970349:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309351661 00000100:00000001:0.0:1788573987.970354:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573987.970356:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.970358:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573987.970361:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573987/real 1788573987] req@ffff8fc4901a4700 x1875440516641280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573987.970367:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573987.970368:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573987.970369:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573987.970371:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573987.970372:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573987.970374:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573987.970376:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516641280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573987.970381:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516641280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970385:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573987.970386:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.970387:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573987.970388:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573987.970390:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516641280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970394:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573987.970397:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516641280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970402:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573987.970404:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.970405:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573987.970406:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.970407:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573987.970409:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573987.970413:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573987.970416:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573987.970418:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573987.970421:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516641280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573987.970426:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516641280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573987.970429:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573987.970431:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516641408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970435:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573987.970436:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573987.970437:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573987.970438:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573987.970440:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573987.970441:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516641408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573987.970443:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573987.970445:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573987.970446:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573987.970447:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573987.970449:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573987.970450:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573987.970453:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573987.970454:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573987.970457:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573987.970459:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516641408, portal 10 00000100:00000001:0.0:1788573987.970460:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573987.970461:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788573987.970466:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516641408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970479:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573987.970481:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573987.970482:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573987.970484:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516641408, offset 0 00000400:00000200:0.0:1788573987.970487:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573987.970490:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573987.970492:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573987.970495:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573987.970496:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573987.970498:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573987.970500:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573987.970502:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546663:546663:256:4294967295] 192.168.202.128@tcp LPNI seq info [546586:546586:8:4294967295] 00000400:00000200:0.0:1788573987.970505:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573987.970510:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573987.970512:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573987.970514:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573987.970517:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573987.970519:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788573987.970526:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573987.970528:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573987.970530:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573987.970532:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573987.970534:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573987.970536:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573987.970537:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573987.970555:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516641408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970561:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573987.970563:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516641408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970576:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.970577:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573987.970580:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573987.970583:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573987.970585:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573987.970587:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309347866 00000100:00000001:0.0:1788573987.970589:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309351661 00000100:00000001:0.0:1788573987.970591:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309347866 00000100:00000001:0.0:1788573987.970592:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309351661 00000100:00000001:0.0:1788573987.970593:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573987.970595:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.970596:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573987.970598:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573987/real 1788573987] req@ffff8fc4901a7480 x1875440516641408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573987.970603:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573987.970605:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573987.970606:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573987.970607:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573987.970609:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573987.970610:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573987.970612:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516641408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573987.970618:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516641408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970622:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573987.970623:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.970624:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573987.970624:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573987.970627:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516641408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970634:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573987.970637:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516641408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970642:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573987.970643:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.970644:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573987.970645:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.970646:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573987.970648:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573987.970650:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573987.970652:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573987.970653:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573987.970656:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516641408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573987.970661:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516641408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573987.970663:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573987.970666:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516641536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970670:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573987.970671:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573987.970672:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573987.970673:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573987.970675:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573987.970676:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516641536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573987.970678:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573987.970679:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573987.970685:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573987.970686:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573987.970689:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573987.970690:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573987.970693:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573987.970695:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573987.970698:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573987.970700:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516641536, portal 4 00000100:00000001:0.0:1788573987.970701:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573987.970702:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788573987.970706:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516641536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970711:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573987.970713:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573987.970714:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573987.970716:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516641536, offset 0 00000400:00000200:0.0:1788573987.970718:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573987.970720:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573987.970723:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573987.970725:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573987.970726:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573987.970728:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573987.970729:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573987.970732:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546664:546664:256:4294967295] 192.168.202.128@tcp LPNI seq info [546587:546587:8:4294967295] 00000400:00000200:0.0:1788573987.970735:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573987.970739:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573987.970742:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573987.970745:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573987.970747:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573987.970750:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788573987.970757:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573987.970762:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573987.970764:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573987.970766:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573987.970769:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573987.970770:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573987.970772:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573987.970774:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516641536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970780:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573987.970782:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516641536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970786:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.970787:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573987.970790:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573987.970792:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573987.970794:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573987.970795:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309348006 00000100:00000001:0.0:1788573987.970797:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309351661 00000100:00000001:0.0:1788573987.970799:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309348006 00000100:00000001:0.0:1788573987.970801:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309351661 00000100:00000001:0.0:1788573987.970802:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573987.970803:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.970804:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573987.970824:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573987/real 1788573987] req@ffff8fc4901a6300 x1875440516641536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573987.970829:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573987.970830:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573987.970831:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573987.970833:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573987.970834:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573987.970852:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573987.970855:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516641536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573987.970861:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516641536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970866:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573987.970867:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.970868:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573987.970870:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573987.970872:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516641536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970876:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573987.970879:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516641536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970883:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573987.970885:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.970886:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573987.970886:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.970888:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573987.970889:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573987.970891:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573987.970893:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573987.970894:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573987.970896:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516641536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573987.970901:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516641536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573987.970903:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573987.970905:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516641664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970915:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573987.970916:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573987.970918:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573987.970919:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573987.970920:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573987.970922:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516641664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573987.970924:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573987.970926:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573987.970928:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573987.970929:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573987.970931:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573987.970932:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573987.970935:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573987.970936:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573987.970939:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573987.970941:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516641664, portal 4 00000100:00000001:0.0:1788573987.970942:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573987.970943:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788573987.970947:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516641664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.970952:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573987.970953:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573987.970955:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573987.970956:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516641664, offset 0 00000400:00000200:0.0:1788573987.970959:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573987.970961:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573987.970963:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573987.970965:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573987.970966:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573987.970968:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573987.970970:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573987.970975:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546665:546665:256:4294967295] 192.168.202.128@tcp LPNI seq info [546588:546588:8:4294967295] 00000400:00000200:0.0:1788573987.970978:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573987.970983:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573987.970986:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573987.970989:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573987.970991:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573987.970993:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788573987.971001:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573987.971002:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573987.971004:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573987.971007:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573987.971009:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573987.971010:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573987.971012:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573987.971014:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516641664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.971019:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573987.971021:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516641664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.971026:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.971027:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573987.971029:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573987.971031:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573987.971034:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573987.971035:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309348009 00000100:00000001:0.0:1788573987.971037:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309351661 00000100:00000001:0.0:1788573987.971038:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309348009 00000100:00000001:0.0:1788573987.971040:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309351661 00000100:00000001:0.0:1788573987.971042:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573987.971046:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.971047:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573987.971050:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573987/real 1788573987] req@ffff8fc4901a4e00 x1875440516641664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573987.971056:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573987.971057:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573987.971058:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573987.971060:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573987.971061:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573987.971062:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573987.971064:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516641664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573987.971069:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516641664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.971074:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573987.971075:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.971076:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573987.971077:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573987.971079:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516641664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.971083:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573987.971085:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516641664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.971089:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573987.971090:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.971091:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573987.971092:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.971093:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573987.971094:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573987.971096:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573987.971101:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573987.971102:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573987.971105:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516641664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573987.971110:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516641664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573987.971112:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.971114:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573987.971115:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516641280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.971121:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573987.971123:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573987.971124:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573987.971127:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573987.971129:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573987.971130:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573987.971132:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573987.971134:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788573987.971137:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573987.971138:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573987.971139:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788573987.971144:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573987.971145:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.971147:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573987.971151:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516641408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.971159:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573987.971160:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573987.971162:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573987.971166:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573987.971168:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573987.971169:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573987.971171:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573987.971178:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788573987.971180:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573987.971181:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573987.971182:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788573987.971185:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573987.971186:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.971188:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573987.971191:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516641536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.971195:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573987.971196:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573987.971197:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573987.971199:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573987.971200:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573987.971200:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573987.971201:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573987.971202:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788573987.971203:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573987.971204:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573987.971205:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788573987.971208:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573987.971209:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.971210:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573987.971212:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516641664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573987.971216:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573987.971217:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573987.971218:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573987.971220:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573987.971220:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573987.971221:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573987.971222:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573987.971223:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5c00. 02000000:00000001:0.0:1788573987.971224:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573987.971225:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573987.971225:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788573987.971231:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573987.971232:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.971233:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573987.971236:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573987.971237:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573987.971238:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573987.971239:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573988.480999:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573988.481013:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573988.481017:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573988.481034:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573988.481044:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573988.481056:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573988.481076:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573988.481080:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573988.481085:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573989.185024:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573989.185029:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573989.185032:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573989.504984:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573989.504998:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573989.505003:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573989.505013:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573989.505025:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573989.505032:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573989.505057:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573989.505064:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573989.505073:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573990.528950:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573990.528956:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573990.528961:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573990.528966:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573990.528976:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573990.528981:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573990.528991:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573990.528995:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573990.529001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573990.529042:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573990.529063:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573990.529067:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573990.529070:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573990.529073:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573990.529075:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573990.529077:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573990.529079:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573990.529090:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573991.553018:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573991.553032:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573991.553042:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573991.553056:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573991.553067:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573991.553073:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573991.553106:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573991.553117:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573991.553126:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573991.808883:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309351661, at 4309351680 00000100:00000010:0.0:1788573991.808904:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573992.257035:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573992.257041:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573992.257045:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573993.089187:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573993.089193:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573993.089198:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573993.089206:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516641792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.089220:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573993.089225:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573993.089231:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573993.089236:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573993.089240:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573993.089247:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516641792:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573993.089253:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573993.089258:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573993.089271:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573993.089276:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573993.089283:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573993.089288:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573993.089299:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573993.089307:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573993.089319:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573993.089328:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516641792, portal 25 00000100:00000001:0.0:1788573993.089336:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573993.089340:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788573993.089352:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516641792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.089366:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573993.089372:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573993.089377:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573993.089382:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516641792, offset 0 00000400:00000200:0.0:1788573993.089396:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573993.089408:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573993.089420:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573993.089430:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573993.089438:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573993.089446:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573993.089454:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573993.089465:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546666:546666:256:4294967295] 192.168.202.128@tcp LPNI seq info [546589:546589:8:4294967295] 00000400:00000200:0.0:1788573993.089477:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573993.089500:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573993.089512:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573993.089522:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573993.089534:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573993.089542:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788573993.089608:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573993.089620:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573993.089629:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573993.089639:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573993.089649:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573993.089654:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573993.089663:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573993.089673:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516641792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.089700:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573993.089708:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516641792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.089722:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.089725:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573993.089732:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573993.089742:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573993.089748:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309356781 00000100:00000010:0.0:1788573993.089756:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573993.089767:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573993.089778:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309356781 00000100:00000010:0.0:1788573993.089784:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50a80. 00000100:00000001:0.0:1788573993.089790:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573993.089795:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573993.089801:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.089841:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573993.089852:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573993/real 1788573993] req@ffff8fc4901a4e00 x1875440516641792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573993.089871:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573993.089875:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573993.089880:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573993.089887:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573993.089892:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573993.089897:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573993.089911:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516641792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573993.089926:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516641792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.089943:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573993.089947:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.089954:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573993.089958:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573993.089969:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516641792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.089988:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573993.089998:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516641792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090031:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573993.090038:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.090043:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573993.090047:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.090053:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573993.090057:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573993.090064:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573993.090075:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573993.090080:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573993.090091:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516641792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573993.090109:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516641792:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573993.090122:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.090128:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573993.090138:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516641792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090177:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573993.090181:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573993.090187:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573993.090194:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573993.090199:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573993.090203:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573993.090210:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573993.090215:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788573993.090221:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573993.090225:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573993.090230:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788573993.090240:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573993.090244:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.090250:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.090261:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573993.090265:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.090270:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573993.090273:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573993.090275:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573993.090284:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516641920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090297:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573993.090301:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573993.090304:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573993.090306:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573993.090309:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573993.090314:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516641920:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573993.090319:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573993.090322:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573993.090325:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573993.090327:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573993.090331:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573993.090335:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573993.090340:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573993.090350:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573993.090355:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573993.090358:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516641920, portal 10 00000100:00000001:0.0:1788573993.090361:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573993.090364:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788573993.090372:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516641920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090380:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573993.090383:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573993.090385:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573993.090388:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516641920, offset 0 00000400:00000200:0.0:1788573993.090392:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573993.090397:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573993.090408:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573993.090412:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573993.090415:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573993.090417:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573993.090420:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573993.090423:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546667:546667:256:4294967295] 192.168.202.128@tcp LPNI seq info [546590:546590:8:4294967295] 00000400:00000200:0.0:1788573993.090427:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573993.090435:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573993.090440:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573993.090444:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573993.090448:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573993.090452:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788573993.090463:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573993.090465:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573993.090469:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573993.090472:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573993.090476:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573993.090478:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573993.090487:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573993.090492:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516641920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090500:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573993.090504:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516641920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090511:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.090513:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573993.090517:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573993.090521:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573993.090525:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573993.090527:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309352979 00000100:00000001:0.0:1788573993.090530:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309356781 00000100:00000001:0.0:1788573993.090532:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309352979 00000100:00000001:0.0:1788573993.090535:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309356781 00000100:00000001:0.0:1788573993.090537:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573993.090539:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.090541:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573993.090546:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573993/real 1788573993] req@ffff8fc4901a6300 x1875440516641920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573993.090555:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573993.090571:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573993.090573:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573993.090576:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573993.090578:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573993.090581:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573993.090584:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516641920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573993.090594:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516641920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090613:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573993.090615:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.090617:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573993.090618:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573993.090623:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516641920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090631:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573993.090635:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516641920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090644:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573993.090647:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.090649:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573993.090650:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.090652:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573993.090655:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573993.090657:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573993.090661:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573993.090664:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573993.090668:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516641920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573993.090677:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516641920:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573993.090682:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573993.090686:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516642048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090693:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573993.090694:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573993.090705:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573993.090707:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573993.090709:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573993.090714:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642048:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573993.090718:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573993.090720:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573993.090721:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573993.090723:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573993.090727:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573993.090729:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573993.090734:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573993.090737:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573993.090741:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573993.090744:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516642048, portal 10 00000100:00000001:0.0:1788573993.090746:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573993.090748:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788573993.090754:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516642048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090760:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573993.090762:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573993.090765:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573993.090767:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516642048, offset 0 00000400:00000200:0.0:1788573993.090771:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573993.090776:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573993.090779:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573993.090783:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573993.090786:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573993.090788:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573993.090791:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573993.090794:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546668:546668:256:4294967295] 192.168.202.128@tcp LPNI seq info [546591:546591:8:4294967295] 00000400:00000200:0.0:1788573993.090799:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573993.090805:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573993.090835:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573993.090838:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573993.090843:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573993.090845:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788573993.090857:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573993.090859:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573993.090862:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573993.090866:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573993.090870:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573993.090872:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573993.090874:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573993.090878:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516642048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090887:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573993.090890:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516642048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090897:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.090899:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573993.090903:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573993.090906:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573993.090910:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573993.090912:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309352987 00000100:00000001:0.0:1788573993.090915:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309356781 00000100:00000001:0.0:1788573993.090917:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309352987 00000100:00000001:0.0:1788573993.090919:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309356781 00000100:00000001:0.0:1788573993.090921:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573993.090924:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.090926:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573993.090929:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573993/real 1788573993] req@ffff8fc4901a7480 x1875440516642048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573993.090943:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573993.090944:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573993.090946:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573993.090949:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573993.090950:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573993.090953:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573993.090956:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516642048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573993.090965:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516642048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090972:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573993.090973:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.090975:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573993.090976:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573993.090981:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516642048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090987:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573993.090991:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516642048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.090998:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573993.091000:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091002:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573993.091004:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091005:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573993.091008:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573993.091011:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573993.091015:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573993.091018:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573993.091027:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516642048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573993.091035:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642048:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573993.091040:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573993.091043:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516642176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091050:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573993.091051:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573993.091054:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573993.091055:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573993.091058:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573993.091061:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642176:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573993.091065:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573993.091067:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573993.091069:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573993.091071:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573993.091074:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573993.091076:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573993.091080:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573993.091083:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573993.091087:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573993.091091:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516642176, portal 4 00000100:00000001:0.0:1788573993.091093:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573993.091095:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788573993.091101:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516642176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091107:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573993.091110:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573993.091113:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573993.091116:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516642176, offset 0 00000400:00000200:0.0:1788573993.091126:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573993.091130:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573993.091134:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573993.091138:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573993.091140:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573993.091143:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573993.091146:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573993.091149:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546669:546669:256:4294967295] 192.168.202.128@tcp LPNI seq info [546592:546592:8:4294967295] 00000400:00000200:0.0:1788573993.091154:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573993.091161:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573993.091166:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573993.091169:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573993.091173:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573993.091176:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788573993.091188:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573993.091190:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573993.091193:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573993.091197:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573993.091200:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573993.091202:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573993.091205:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573993.091209:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516642176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091217:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573993.091220:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516642176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091228:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.091229:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573993.091232:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573993.091236:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573993.091239:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573993.091248:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309353127 00000100:00000001:0.0:1788573993.091250:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309356781 00000100:00000001:0.0:1788573993.091253:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309353127 00000100:00000001:0.0:1788573993.091255:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309356781 00000100:00000001:0.0:1788573993.091257:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573993.091260:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.091262:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573993.091267:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573993/real 1788573993] req@ffff8fc4901a4700 x1875440516642176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573993.091275:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573993.091277:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573993.091279:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573993.091281:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573993.091283:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573993.091285:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573993.091289:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516642176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573993.091297:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516642176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091304:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573993.091306:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091308:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573993.091309:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573993.091313:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516642176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091320:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573993.091325:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516642176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091331:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573993.091340:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091342:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573993.091344:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091345:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573993.091348:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573993.091350:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573993.091354:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573993.091357:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573993.091361:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516642176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573993.091370:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642176:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573993.091374:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091376:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573993.091380:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516641920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091387:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573993.091388:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573993.091391:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573993.091393:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573993.091395:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573993.091396:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573993.091399:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573993.091400:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788573993.091403:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573993.091405:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573993.091406:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788573993.091411:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573993.091412:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091414:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573993.091417:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516642048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091436:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573993.091438:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573993.091440:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788573993.091442:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573993.091444:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573993.091445:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573993.091447:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573993.091449:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5c00. 02000000:00000001:0.0:1788573993.091451:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573993.091452:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573993.091454:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788573993.091457:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573993.091458:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091460:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573993.091464:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516642176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091471:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573993.091472:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573993.091474:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573993.091476:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573993.091478:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573993.091479:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573993.091481:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573993.091482:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788573993.091484:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573993.091485:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573993.091487:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788573993.091489:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573993.091491:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091493:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091497:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573993.091498:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.091500:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573993.091502:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573993.091503:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573993.091513:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516642304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091520:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573993.091522:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573993.091524:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573993.091526:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573993.091528:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573993.091532:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642304:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573993.091535:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573993.091537:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573993.091539:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573993.091541:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573993.091544:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573993.091546:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573993.091550:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573993.091552:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573993.091571:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573993.091577:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516642304, portal 4 00000100:00000001:0.0:1788573993.091580:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573993.091581:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788573993.091588:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516642304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091594:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573993.091597:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573993.091600:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573993.091602:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516642304, offset 0 00000400:00000200:0.0:1788573993.091607:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573993.091611:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573993.091615:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573993.091618:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573993.091621:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573993.091624:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573993.091626:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573993.091635:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546670:546670:256:4294967295] 192.168.202.128@tcp LPNI seq info [546593:546593:8:4294967295] 00000400:00000200:0.0:1788573993.091640:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573993.091647:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573993.091652:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573993.091655:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573993.091659:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573993.091662:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788573993.091674:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573993.091676:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573993.091679:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573993.091683:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573993.091686:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573993.091688:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573993.091691:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573993.091695:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516642304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091703:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573993.091707:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516642304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091714:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.091716:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573993.091720:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573993.091723:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573993.091726:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573993.091728:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309353129 00000100:00000001:0.0:1788573993.091731:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309356781 00000100:00000001:0.0:1788573993.091733:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309353129 00000100:00000001:0.0:1788573993.091735:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309356781 00000100:00000001:0.0:1788573993.091737:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573993.091745:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.091747:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573993.091752:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573993/real 1788573993] req@ffff8fc4901a5880 x1875440516642304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573993.091761:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573993.091762:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573993.091764:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573993.091767:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573993.091768:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573993.091771:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573993.091774:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516642304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573993.091782:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516642304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091788:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573993.091790:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091792:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573993.091793:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573993.091798:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516642304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091804:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573993.091826:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516642304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091834:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573993.091836:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091838:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573993.091839:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091841:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573993.091844:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573993.091846:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573993.091855:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573993.091857:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573993.091862:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516642304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573993.091870:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642304:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573993.091874:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091876:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573993.091879:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516642304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573993.091885:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573993.091886:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573993.091888:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788573993.091891:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573993.091892:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573993.091894:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573993.091896:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573993.091898:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788573993.091900:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573993.091901:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573993.091903:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788573993.091906:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573993.091907:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091909:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.091913:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573993.091914:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.091916:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573993.091917:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.600962:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573993.600967:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573993.600970:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573993.600974:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788573993.600978:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573993.600981:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.600996:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573993.600998:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.601000:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573993.601080:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573993.601086:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573993.601089:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573993.601095:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788573993.601098:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573993.601100:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.601103:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573993.601104:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573993.601107:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573994.305005:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573994.305008:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573994.305011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573994.625125:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573994.625145:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573994.625150:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573994.625157:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573994.625164:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573994.625168:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573994.625261:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573994.625264:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573994.625267:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573995.648991:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573995.649004:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573995.649011:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573995.649027:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573995.649042:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573995.649048:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573995.649075:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573995.649078:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573995.649081:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788573996.161196:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788573996.672920:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573996.672929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573996.672934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573996.672943:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573996.672976:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573996.672987:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573996.673020:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573996.673028:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573996.673033:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573996.928926:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309356781, at 4309356800 00000100:00000010:0.0:1788573996.928944:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788573997.377012:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573997.377016:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573997.377018:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573997.696941:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573997.696950:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573997.696960:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573997.696981:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573997.696988:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573997.697000:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573997.697017:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573997.697029:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573997.697036:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.209265:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573998.209274:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573998.209278:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573998.209293:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516642432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.209313:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573998.209318:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573998.209323:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573998.209328:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573998.209333:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573998.209337:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516642432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573998.209340:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573998.209348:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573998.209351:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573998.209355:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573998.209366:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573998.209386:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573998.209405:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573998.209410:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573998.209418:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573998.209431:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516642432, portal 25 00000100:00000001:0.0:1788573998.209437:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573998.209441:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788573998.209455:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516642432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.209479:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573998.209484:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573998.209492:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573998.209500:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516642432, offset 0 00000400:00000200:0.0:1788573998.209519:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573998.209537:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573998.209552:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573998.209564:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573998.209588:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573998.209598:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573998.209611:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573998.209623:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546671:546671:256:4294967295] 192.168.202.128@tcp LPNI seq info [546594:546594:8:4294967295] 00000400:00000200:0.0:1788573998.209638:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573998.209665:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573998.209684:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573998.209694:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573998.209708:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573998.209717:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788573998.209760:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573998.209774:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573998.209792:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573998.209842:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573998.209868:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573998.209873:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573998.209890:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573998.209909:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516642432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.209946:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573998.209957:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516642432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.209979:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.209985:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573998.209999:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573998.210011:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573998.210019:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309361902 00000100:00000010:0.0:1788573998.210031:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788573998.210045:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788573998.210061:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309361902 00000100:00000010:0.0:1788573998.210069:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50300. 00000100:00000001:0.0:1788573998.210081:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788573998.210092:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573998.210101:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.210110:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573998.210128:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573998/real 1788573998] req@ffff8fc4901a7100 x1875440516642432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573998.210161:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573998.210168:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573998.210174:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573998.210184:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573998.210191:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573998.210202:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573998.210214:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516642432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573998.210260:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516642432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.210315:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573998.210320:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.210328:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573998.210334:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573998.210347:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516642432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.210372:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573998.210382:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516642432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.210412:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573998.210421:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.210425:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573998.210427:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.210433:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573998.210441:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573998.210451:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573998.210458:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573998.210461:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573998.210466:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516642432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573998.210475:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516642432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788573998.210480:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.210483:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573998.210485:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516642432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.210492:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573998.210494:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573998.210507:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573998.210511:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573998.210514:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573998.210517:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788573998.210521:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573998.210524:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788573998.210529:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573998.210531:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573998.210534:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788573998.210539:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573998.210541:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.210544:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.210552:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573998.210554:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.210557:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573998.210559:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573998.210562:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573998.210567:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516642560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.210589:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573998.210591:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573998.210594:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573998.210597:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573998.210599:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573998.210604:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573998.210608:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573998.210610:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573998.210612:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573998.210614:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573998.210619:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573998.210622:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573998.210628:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573998.210640:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573998.210650:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573998.210660:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516642560, portal 10 00000100:00000001:0.0:1788573998.210675:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573998.210679:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788573998.210695:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516642560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.210718:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573998.210731:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573998.210743:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573998.210754:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516642560, offset 0 00000400:00000200:0.0:1788573998.210773:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573998.210790:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573998.210871:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573998.210887:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573998.210897:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573998.210911:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573998.210921:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573998.210936:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546672:546672:256:4294967295] 192.168.202.128@tcp LPNI seq info [546595:546595:8:4294967295] 00000400:00000200:0.0:1788573998.210953:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573998.210980:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573998.210999:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573998.211010:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573998.211026:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573998.211036:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788573998.211084:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573998.211097:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573998.211110:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573998.211125:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573998.211135:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573998.211144:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573998.211155:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573998.211166:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516642560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211202:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573998.211206:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516642560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211211:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.211212:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573998.211217:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573998.211221:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788573998.211224:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573998.211226:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309358100 00000100:00000001:0.0:1788573998.211229:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309361902 00000100:00000001:0.0:1788573998.211231:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309358100 00000100:00000001:0.0:1788573998.211233:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309361902 00000100:00000001:0.0:1788573998.211235:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573998.211238:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.211241:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573998.211248:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573998/real 1788573998] req@ffff8fc4901a5c00 x1875440516642560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573998.211258:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573998.211260:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573998.211264:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573998.211269:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573998.211271:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573998.211276:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573998.211282:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516642560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573998.211292:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516642560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211300:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573998.211308:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.211311:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573998.211313:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573998.211318:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516642560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211325:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573998.211330:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516642560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211338:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573998.211341:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.211342:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573998.211344:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.211346:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573998.211348:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573998.211350:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573998.211353:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573998.211355:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573998.211358:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516642560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573998.211364:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573998.211367:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573998.211369:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516642688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211373:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573998.211374:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573998.211376:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573998.211379:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573998.211381:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573998.211389:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573998.211392:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573998.211394:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573998.211396:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573998.211397:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573998.211400:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573998.211402:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573998.211405:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573998.211407:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573998.211409:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573998.211412:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516642688, portal 10 00000100:00000001:0.0:1788573998.211414:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573998.211415:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788573998.211419:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516642688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211423:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573998.211425:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573998.211426:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573998.211428:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516642688, offset 0 00000400:00000200:0.0:1788573998.211431:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573998.211434:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573998.211437:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573998.211439:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573998.211441:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573998.211443:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573998.211444:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573998.211447:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546673:546673:256:4294967295] 192.168.202.128@tcp LPNI seq info [546596:546596:8:4294967295] 00000400:00000200:0.0:1788573998.211449:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573998.211455:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573998.211458:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573998.211460:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573998.211466:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573998.211468:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788573998.211478:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573998.211480:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573998.211482:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573998.211484:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573998.211487:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573998.211488:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573998.211490:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573998.211492:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516642688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211500:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573998.211503:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516642688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211510:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.211512:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573998.211516:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573998.211519:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788573998.211523:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573998.211524:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309358107 00000100:00000001:0.0:1788573998.211527:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309361902 00000100:00000001:0.0:1788573998.211530:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309358107 00000100:00000001:0.0:1788573998.211532:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309361902 00000100:00000001:0.0:1788573998.211534:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573998.211536:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.211538:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573998.211542:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573998/real 1788573998] req@ffff8fc4901a5180 x1875440516642688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573998.211551:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573998.211562:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573998.211564:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573998.211567:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573998.211569:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573998.211572:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573998.211590:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516642688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573998.211600:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516642688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211607:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573998.211609:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.211611:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573998.211613:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573998.211617:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516642688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211624:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573998.211628:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516642688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211636:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573998.211638:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.211640:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573998.211641:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.211643:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573998.211646:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573998.211649:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573998.211652:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573998.211654:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573998.211658:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516642688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573998.211678:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788573998.211682:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.211684:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573998.211687:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516642560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211694:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573998.211696:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573998.211698:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573998.211702:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573998.211704:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573998.211706:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573998.211708:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573998.211710:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6400. 02000000:00000001:0.0:1788573998.211713:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573998.211715:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573998.211717:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788573998.211722:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573998.211723:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.211724:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573998.211727:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516642688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211732:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573998.211732:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573998.211733:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573998.211735:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573998.211736:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573998.211736:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788573998.211738:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573998.211738:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7000. 02000000:00000001:0.0:1788573998.211740:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573998.211741:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573998.211741:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788573998.211748:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573998.211748:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.211750:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.211754:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573998.211755:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.211756:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573998.211758:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573998.211759:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573998.211763:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516642816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211768:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573998.211769:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573998.211771:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573998.211772:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573998.211774:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573998.211775:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573998.211778:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573998.211780:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573998.211781:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573998.211782:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573998.211784:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573998.211786:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573998.211789:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573998.211791:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573998.211793:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573998.211795:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516642816, portal 4 00000100:00000001:0.0:1788573998.211797:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573998.211798:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788573998.211802:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516642816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211823:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573998.211825:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573998.211827:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573998.211829:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516642816, offset 0 00000400:00000200:0.0:1788573998.211836:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573998.211839:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573998.211841:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573998.211843:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573998.211845:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573998.211847:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573998.211849:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573998.211851:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546674:546674:256:4294967295] 192.168.202.128@tcp LPNI seq info [546597:546597:8:4294967295] 00000400:00000200:0.0:1788573998.211853:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573998.211858:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573998.211861:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573998.211864:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573998.211866:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573998.211868:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788573998.211876:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573998.211877:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573998.211879:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573998.211882:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573998.211884:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573998.211885:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573998.211887:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573998.211889:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516642816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211895:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573998.211897:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516642816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211901:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.211902:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573998.211904:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573998.211907:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788573998.211909:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573998.211915:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309358247 00000100:00000001:0.0:1788573998.211917:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309361902 00000100:00000001:0.0:1788573998.211919:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309358247 00000100:00000001:0.0:1788573998.211920:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309361902 00000100:00000001:0.0:1788573998.211922:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573998.211923:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.211925:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573998.211929:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573998/real 1788573998] req@ffff8fc4901a7800 x1875440516642816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573998.211938:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573998.211939:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573998.211941:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573998.211943:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573998.211946:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573998.211948:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573998.211952:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516642816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573998.211960:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516642816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211968:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573998.211969:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.211971:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573998.211973:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573998.211977:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516642816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211984:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573998.211988:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516642816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.211995:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573998.212003:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.212005:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573998.212007:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.212008:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573998.212010:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573998.212013:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573998.212018:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573998.212019:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573998.212022:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516642816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573998.212028:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573998.212031:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788573998.212033:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516642944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.212038:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573998.212039:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573998.212041:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788573998.212042:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788573998.212043:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573998.212045:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573998.212047:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788573998.212049:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788573998.212050:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788573998.212051:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788573998.212054:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573998.212055:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788573998.212057:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788573998.212060:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788573998.212063:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788573998.212067:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516642944, portal 4 00000100:00000001:0.0:1788573998.212074:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788573998.212076:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788573998.212082:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516642944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.212089:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788573998.212092:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788573998.212095:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788573998.212098:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516642944, offset 0 00000400:00000200:0.0:1788573998.212102:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788573998.212106:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573998.212110:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788573998.212113:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788573998.212116:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788573998.212119:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788573998.212121:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788573998.212124:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546675:546675:256:4294967295] 192.168.202.128@tcp LPNI seq info [546598:546598:8:4294967295] 00000400:00000200:0.0:1788573998.212128:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788573998.212135:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788573998.212140:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573998.212144:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788573998.212148:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788573998.212151:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788573998.212162:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788573998.212165:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788573998.212167:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788573998.212172:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788573998.212175:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788573998.212177:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788573998.212180:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788573998.212183:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516642944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.212202:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573998.212206:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516642944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.212212:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.212214:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788573998.212218:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788573998.212221:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788573998.212225:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788573998.212227:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309358249 00000100:00000001:0.0:1788573998.212230:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309361902 00000100:00000001:0.0:1788573998.212232:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309358249 00000100:00000001:0.0:1788573998.212234:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309361902 00000100:00000001:0.0:1788573998.212236:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788573998.212239:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.212241:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788573998.212245:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788573998/real 1788573998] req@ffff8fc4901a4a80 x1875440516642944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788573998.212253:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788573998.212255:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788573998.212257:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788573998.212259:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788573998.212261:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788573998.212264:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788573998.212267:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516642944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788573998.212276:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516642944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.212282:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788573998.212288:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.212290:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573998.212292:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788573998.212296:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516642944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.212303:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788573998.212307:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516642944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.212315:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788573998.212317:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.212319:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788573998.212321:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.212323:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788573998.212325:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788573998.212328:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788573998.212332:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788573998.212334:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788573998.212339:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516642944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788573998.212347:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516642944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788573998.212352:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.212354:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573998.212357:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516642816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.212363:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573998.212365:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573998.212367:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788573998.212370:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573998.212371:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573998.212378:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573998.212380:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573998.212382:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7400. 02000000:00000001:0.0:1788573998.212385:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573998.212386:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573998.212388:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788573998.212391:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573998.212393:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.212395:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788573998.212400:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516642944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788573998.212407:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788573998.212409:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788573998.212411:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788573998.212414:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788573998.212416:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788573998.212418:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788573998.212420:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788573998.212422:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788573998.212424:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788573998.212426:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788573998.212427:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788573998.212431:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788573998.212432:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.212434:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.212439:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573998.212440:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.212442:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788573998.212444:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788573998.720899:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573998.720907:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573998.720911:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573998.720920:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788573998.720926:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573998.720933:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.720950:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573998.720954:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573998.720978:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788573999.425042:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788573999.425045:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788573999.425049:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788573999.744977:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788573999.744986:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788573999.744990:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788573999.744995:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788573999.744999:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788573999.745004:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573999.745008:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788573999.745010:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788573999.745014:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574000.768941:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574000.768951:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574000.768953:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574000.768958:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574000.768964:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574000.768967:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574000.768989:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574000.768993:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574000.768999:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574001.792955:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574001.792969:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574001.792977:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574001.792991:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574001.792997:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574001.793001:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574001.793037:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574001.793044:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574001.793050:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574002.048941:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309361902, at 4309361920 00000100:00000010:0.0:1788574002.048959:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788574002.497018:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574002.497023:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574002.497026:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574002.816987:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574002.817018:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574002.817022:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574002.817029:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574002.817043:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574002.817051:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574002.817064:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574002.817070:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574002.817076:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.329296:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574003.329304:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574003.329308:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574003.329319:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5f80 x1875440516643072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.329334:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574003.329337:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574003.329341:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574003.329344:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574003.329348:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574003.329356:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516643072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574003.329365:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574003.329370:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574003.329375:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574003.329380:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574003.329388:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574003.329394:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574003.329405:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574003.329411:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574003.329421:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574003.329431:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516643072, portal 25 00000100:00000001:0.0:1788574003.329436:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574003.329439:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605328768 : -123400104222848 : ffff8fc4ac9f5f80) 00000100:00000040:0.0:1788574003.329448:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5f80 x1875440516643072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.329463:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574003.329480:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574003.329485:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574003.329493:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516643072, offset 0 00000400:00000200:0.0:1788574003.329507:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574003.329524:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574003.329539:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574003.329550:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574003.329557:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574003.329565:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574003.329571:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574003.329580:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546676:546676:256:4294967295] 192.168.202.128@tcp LPNI seq info [546599:546599:8:4294967295] 00000400:00000200:0.0:1788574003.329589:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574003.329614:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574003.329626:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574003.329633:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574003.329646:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574003.329655:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788574003.329699:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574003.329705:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574003.329725:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574003.329739:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574003.329753:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574003.329758:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574003.329768:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574003.329778:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5f80 x1875440516643072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.329805:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574003.329848:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5f80 x1875440516643072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.329865:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.329868:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574003.329883:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574003.329890:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574003.329893:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309367022 00000100:00000010:0.0:1788574003.329901:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788574003.329911:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574003.329922:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309367022 00000100:00000010:0.0:1788574003.329929:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50e40. 00000100:00000001:0.0:1788574003.329937:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574003.329945:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574003.329953:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.329960:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574003.329968:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574003/real 1788574003] req@ffff8fc4ac9f5f80 x1875440516643072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574003.329987:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574003.329989:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574003.329993:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574003.329997:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574003.330002:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574003.330006:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574003.330013:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5f80 x1875440516643072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574003.330028:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5f80 x1875440516643072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330044:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574003.330049:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.330055:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574003.330061:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574003.330073:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5f80 x1875440516643072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330099:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574003.330107:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5f80 x1875440516643072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330124:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574003.330128:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.330132:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574003.330136:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.330142:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574003.330144:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574003.330148:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574003.330154:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574003.330158:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574003.330166:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5f80 x1875440516643072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574003.330178:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516643072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574003.330186:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.330191:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574003.330197:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5f80 x1875440516643072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330212:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574003.330215:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574003.330220:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574003.330225:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574003.330228:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574003.330231:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574003.330234:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574003.330236:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34c00. 02000000:00000001:0.0:1788574003.330240:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574003.330241:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574003.330243:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5f80. 00000100:00000001:0.0:1788574003.330245:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574003.330246:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.330248:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.330261:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574003.330263:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.330265:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574003.330267:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574003.330268:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574003.330272:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4700 x1875440516643200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330278:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574003.330279:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574003.330280:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574003.330282:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574003.330283:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574003.330285:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516643200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574003.330288:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574003.330290:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574003.330291:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574003.330292:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574003.330294:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574003.330296:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574003.330299:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574003.330301:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574003.330304:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574003.330307:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516643200, portal 10 00000100:00000001:0.0:1788574003.330308:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574003.330309:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605322496 : -123400104229120 : ffff8fc4ac9f4700) 00000100:00000040:0.0:1788574003.330316:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4700 x1875440516643200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330320:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574003.330322:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574003.330323:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574003.330325:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516643200, offset 0 00000400:00000200:0.0:1788574003.330328:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574003.330331:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574003.330337:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574003.330340:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574003.330342:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574003.330344:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574003.330346:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574003.330349:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546677:546677:256:4294967295] 192.168.202.128@tcp LPNI seq info [546600:546600:8:4294967295] 00000400:00000200:0.0:1788574003.330351:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574003.330357:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574003.330360:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574003.330362:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574003.330365:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574003.330367:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788574003.330376:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574003.330377:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574003.330380:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574003.330382:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574003.330385:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574003.330386:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574003.330388:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574003.330391:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4700 x1875440516643200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330397:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574003.330399:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4700 x1875440516643200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330404:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.330405:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574003.330407:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574003.330410:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574003.330413:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574003.330414:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309363219 00000100:00000001:0.0:1788574003.330420:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309367022 00000100:00000001:0.0:1788574003.330421:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309363219 00000100:00000001:0.0:1788574003.330423:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309367022 00000100:00000001:0.0:1788574003.330425:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574003.330426:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.330427:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574003.330430:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574003/real 1788574003] req@ffff8fc4ac9f4700 x1875440516643200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574003.330435:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574003.330436:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574003.330438:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574003.330439:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574003.330440:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574003.330442:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574003.330444:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4700 x1875440516643200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574003.330450:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4700 x1875440516643200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330454:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574003.330455:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.330456:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574003.330457:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574003.330459:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4700 x1875440516643200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330463:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574003.330466:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4700 x1875440516643200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330470:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574003.330471:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.330477:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574003.330478:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.330479:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574003.330481:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574003.330484:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574003.330488:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574003.330491:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574003.330495:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4700 x1875440516643200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574003.330503:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516643200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574003.330508:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574003.330512:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7800 x1875440516643328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330519:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574003.330521:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574003.330523:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574003.330525:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574003.330528:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574003.330531:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516643328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574003.330535:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574003.330537:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574003.330539:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574003.330540:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574003.330543:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574003.330545:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574003.330548:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574003.330549:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574003.330552:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574003.330554:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516643328, portal 10 00000100:00000001:0.0:1788574003.330556:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574003.330560:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335040 : -123400104216576 : ffff8fc4ac9f7800) 00000100:00000040:0.0:1788574003.330566:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7800 x1875440516643328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330571:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574003.330572:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574003.330574:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574003.330576:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516643328, offset 0 00000400:00000200:0.0:1788574003.330578:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574003.330581:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574003.330583:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574003.330586:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574003.330587:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574003.330589:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574003.330591:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574003.330609:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546678:546678:256:4294967295] 192.168.202.128@tcp LPNI seq info [546601:546601:8:4294967295] 00000400:00000200:0.0:1788574003.330613:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574003.330619:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574003.330622:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574003.330625:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574003.330628:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574003.330630:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788574003.330639:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574003.330642:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574003.330645:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574003.330649:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574003.330653:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574003.330655:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574003.330658:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574003.330662:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7800 x1875440516643328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330684:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574003.330688:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7800 x1875440516643328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330693:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.330695:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574003.330698:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574003.330701:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574003.330704:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574003.330705:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309363226 00000100:00000001:0.0:1788574003.330707:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309367022 00000100:00000001:0.0:1788574003.330709:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309363226 00000100:00000001:0.0:1788574003.330711:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309367022 00000100:00000001:0.0:1788574003.330712:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574003.330713:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.330715:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574003.330717:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574003/real 1788574003] req@ffff8fc4ac9f7800 x1875440516643328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574003.330723:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574003.330724:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574003.330725:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574003.330727:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574003.330729:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574003.330732:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574003.330735:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7800 x1875440516643328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574003.330744:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7800 x1875440516643328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330752:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574003.330758:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.330761:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574003.330762:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574003.330767:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7800 x1875440516643328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330775:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574003.330779:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7800 x1875440516643328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330785:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574003.330786:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.330788:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574003.330788:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.330790:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574003.330792:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574003.330794:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574003.330796:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574003.330797:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574003.330800:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7800 x1875440516643328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574003.330806:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516643328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574003.330825:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574003.330827:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5180 x1875440516643456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330832:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574003.330833:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574003.330835:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574003.330837:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574003.330838:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574003.330845:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516643456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574003.330847:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574003.330849:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574003.330850:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574003.330852:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574003.330854:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574003.330856:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574003.330859:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574003.330861:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574003.330863:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574003.330866:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516643456, portal 4 00000100:00000001:0.0:1788574003.330867:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574003.330868:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605325184 : -123400104226432 : ffff8fc4ac9f5180) 00000100:00000040:0.0:1788574003.330873:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5180 x1875440516643456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330878:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574003.330879:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574003.330880:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574003.330882:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516643456, offset 0 00000400:00000200:0.0:1788574003.330885:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574003.330887:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574003.330890:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574003.330892:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574003.330894:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574003.330895:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574003.330897:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574003.330899:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546679:546679:256:4294967295] 192.168.202.128@tcp LPNI seq info [546602:546602:8:4294967295] 00000400:00000200:0.0:1788574003.330902:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574003.330906:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574003.330911:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574003.330914:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574003.330924:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574003.330927:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788574003.330939:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574003.330941:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574003.330944:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574003.330948:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574003.330952:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574003.330954:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574003.330957:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574003.330961:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5180 x1875440516643456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330967:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574003.330970:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5180 x1875440516643456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.330974:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.330976:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574003.330978:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574003.330981:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574003.330984:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574003.330985:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309363367 00000100:00000001:0.0:1788574003.330987:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309367022 00000100:00000001:0.0:1788574003.330989:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309363367 00000100:00000001:0.0:1788574003.330990:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309367022 00000100:00000001:0.0:1788574003.330992:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574003.330993:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.330994:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574003.330996:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574003/real 1788574003] req@ffff8fc4ac9f5180 x1875440516643456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574003.331002:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574003.331007:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574003.331009:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574003.331010:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574003.331011:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574003.331013:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574003.331016:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516643456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574003.331022:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516643456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331025:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574003.331026:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.331027:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574003.331028:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574003.331031:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516643456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331035:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574003.331037:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516643456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331042:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574003.331044:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.331046:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574003.331047:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.331049:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574003.331051:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574003.331054:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574003.331057:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574003.331060:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574003.331064:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516643456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574003.331088:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516643456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574003.331092:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574003.331095:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516643584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331101:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574003.331102:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574003.331104:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574003.331105:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574003.331107:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574003.331109:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516643584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574003.331111:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574003.331113:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574003.331114:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574003.331115:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574003.331117:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574003.331119:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574003.331121:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574003.331123:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574003.331126:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574003.331128:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516643584, portal 4 00000100:00000001:0.0:1788574003.331130:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574003.331131:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788574003.331135:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516643584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331139:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574003.331141:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574003.331142:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574003.331144:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516643584, offset 0 00000400:00000200:0.0:1788574003.331147:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574003.331153:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574003.331156:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574003.331159:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574003.331161:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574003.331162:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574003.331164:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574003.331166:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546680:546680:256:4294967295] 192.168.202.128@tcp LPNI seq info [546603:546603:8:4294967295] 00000400:00000200:0.0:1788574003.331169:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574003.331174:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574003.331177:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574003.331179:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574003.331182:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574003.331184:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788574003.331192:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574003.331193:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574003.331195:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574003.331198:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574003.331200:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574003.331201:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574003.331203:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574003.331206:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516643584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331211:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574003.331213:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516643584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331218:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.331219:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574003.331221:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574003.331223:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574003.331225:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574003.331230:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309363369 00000100:00000001:0.0:1788574003.331232:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309367022 00000100:00000001:0.0:1788574003.331234:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309363369 00000100:00000001:0.0:1788574003.331236:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309367022 00000100:00000001:0.0:1788574003.331238:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574003.331239:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.331240:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574003.331243:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574003/real 1788574003] req@ffff8fc4ac9f6a00 x1875440516643584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574003.331248:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574003.331249:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574003.331250:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574003.331252:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574003.331253:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574003.331255:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574003.331257:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516643584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574003.331263:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516643584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331266:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574003.331267:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.331268:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574003.331269:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574003.331271:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516643584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331275:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574003.331278:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516643584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331281:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574003.331286:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.331287:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574003.331288:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.331289:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574003.331291:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574003.331293:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574003.331295:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574003.331296:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574003.331299:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516643584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574003.331305:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516643584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574003.331307:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.331308:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574003.331310:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4700 x1875440516643200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331314:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574003.331315:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574003.331316:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574003.331318:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574003.331319:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574003.331320:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574003.331321:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574003.331322:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788574003.331324:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574003.331325:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574003.331326:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4700. 00000100:00000001:0.0:1788574003.331328:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574003.331329:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.331330:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574003.331333:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7800 x1875440516643328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331344:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574003.331345:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574003.331347:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574003.331348:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574003.331349:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574003.331350:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574003.331351:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574003.331352:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35c00. 02000000:00000001:0.0:1788574003.331353:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574003.331354:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574003.331355:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7800. 00000100:00000001:0.0:1788574003.331356:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574003.331357:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.331358:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574003.331360:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516643456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331364:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574003.331365:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574003.331366:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574003.331367:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574003.331368:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574003.331368:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574003.331369:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574003.331370:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35400. 02000000:00000001:0.0:1788574003.331371:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574003.331372:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574003.331373:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788574003.331374:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574003.331375:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.331375:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574003.331377:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516643584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574003.331381:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574003.331382:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574003.331385:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574003.331388:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574003.331389:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574003.331391:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574003.331392:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574003.331393:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36400. 02000000:00000001:0.0:1788574003.331394:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574003.331395:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574003.331396:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788574003.331397:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574003.331398:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.331399:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.331402:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574003.331402:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.331404:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574003.331405:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574003.840938:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574003.840950:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574003.840962:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574003.840983:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574003.840996:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574003.841011:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.841026:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574003.841034:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574003.841040:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574004.545024:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574004.545030:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574004.545034:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574004.864960:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574004.864972:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574004.864975:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574004.864989:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574004.864997:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574004.865004:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574004.865028:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574004.865033:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574004.865039:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574005.888948:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574005.888961:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574005.888965:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574005.888971:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574005.888977:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574005.888983:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574005.888990:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574005.888994:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574005.888998:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574006.912926:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574006.912932:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574006.912937:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574006.912945:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574006.912951:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574006.912955:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574006.912962:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574006.912968:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574006.912972:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574007.168931:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309367022, at 4309367040 00000100:00000010:0.0:1788574007.168949:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788574007.617106:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574007.617110:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574007.617112:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574007.936893:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574007.936898:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574007.936901:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574007.936906:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574007.936911:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574007.936915:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574007.936931:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574007.936936:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574007.936944:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.449179:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574008.449184:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574008.449195:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574008.449204:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4e00 x1875440516643712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.449219:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574008.449231:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574008.449239:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574008.449243:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574008.449246:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574008.449250:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516643712:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574008.449257:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574008.449262:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574008.449265:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574008.449270:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574008.449275:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574008.449278:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574008.449288:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574008.449292:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574008.449300:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574008.449307:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516643712, portal 25 00000100:00000001:0.0:1788574008.449312:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574008.449315:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605324288 : -123400104227328 : ffff8fc4ac9f4e00) 00000100:00000040:0.0:1788574008.449322:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4e00 x1875440516643712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.449332:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574008.449336:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574008.449342:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574008.449346:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516643712, offset 0 00000400:00000200:0.0:1788574008.449354:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574008.449362:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574008.449369:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574008.449375:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574008.449382:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574008.449388:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574008.449393:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574008.449401:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546681:546681:256:4294967295] 192.168.202.128@tcp LPNI seq info [546604:546604:8:4294967295] 00000400:00000200:0.0:1788574008.449408:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574008.449421:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574008.449440:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574008.449445:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574008.449455:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574008.449459:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788574008.449476:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574008.449479:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574008.449487:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574008.449498:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574008.449508:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574008.449513:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574008.449522:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574008.449532:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4e00 x1875440516643712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.449554:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574008.449562:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4e00 x1875440516643712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.449573:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.449578:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574008.449587:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574008.449599:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574008.449605:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309372141 00000100:00000010:0.0:1788574008.449625:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788574008.449634:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574008.449641:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309372141 00000100:00000010:0.0:1788574008.449648:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50de0. 00000100:00000001:0.0:1788574008.449655:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574008.449660:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574008.449666:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.449670:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574008.449675:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574008/real 1788574008] req@ffff8fc4ac9f4e00 x1875440516643712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574008.449705:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574008.449708:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574008.449711:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574008.449717:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574008.449720:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574008.449723:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574008.449734:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4e00 x1875440516643712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574008.449751:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4e00 x1875440516643712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.449762:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574008.449765:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.449771:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574008.449772:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574008.449779:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4e00 x1875440516643712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.449829:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574008.449833:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4e00 x1875440516643712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.449845:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574008.449851:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.449854:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574008.449856:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.449862:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574008.449865:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574008.449868:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574008.449874:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574008.449878:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574008.449883:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4e00 x1875440516643712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574008.449906:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516643712:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574008.449914:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.449918:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574008.449923:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4e00 x1875440516643712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.449934:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574008.449939:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574008.449943:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574008.449956:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574008.449962:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574008.449965:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574008.449970:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574008.449974:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37c00. 02000000:00000001:0.0:1788574008.449978:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574008.449980:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574008.449984:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4e00. 00000100:00000001:0.0:1788574008.449988:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574008.449990:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.449994:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.450001:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574008.450005:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.450008:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574008.450010:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574008.450012:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574008.450018:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7480 x1875440516643840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450032:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574008.450034:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574008.450038:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574008.450042:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574008.450045:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574008.450049:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516643840:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574008.450061:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574008.450064:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574008.450065:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574008.450067:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574008.450073:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574008.450078:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574008.450084:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574008.450088:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574008.450093:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574008.450100:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516643840, portal 10 00000100:00000001:0.0:1788574008.450104:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574008.450105:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605334144 : -123400104217472 : ffff8fc4ac9f7480) 00000100:00000040:0.0:1788574008.450112:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7480 x1875440516643840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450123:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574008.450128:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574008.450132:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574008.450136:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516643840, offset 0 00000400:00000200:0.0:1788574008.450145:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574008.450153:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574008.450158:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574008.450163:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574008.450167:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574008.450169:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574008.450170:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574008.450173:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546682:546682:256:4294967295] 192.168.202.128@tcp LPNI seq info [546605:546605:8:4294967295] 00000400:00000200:0.0:1788574008.450185:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574008.450197:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574008.450202:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574008.450207:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574008.450214:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574008.450228:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788574008.450250:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574008.450253:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574008.450257:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574008.450265:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574008.450271:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574008.450274:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574008.450278:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574008.450285:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7480 x1875440516643840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450299:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574008.450304:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7480 x1875440516643840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450318:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.450322:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574008.450326:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574008.450331:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574008.450338:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574008.450343:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309368339 00000100:00000001:0.0:1788574008.450350:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309372141 00000100:00000001:0.0:1788574008.450355:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309368339 00000100:00000001:0.0:1788574008.450360:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309372141 00000100:00000001:0.0:1788574008.450365:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574008.450368:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.450370:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574008.450373:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574008/real 1788574008] req@ffff8fc4ac9f7480 x1875440516643840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574008.450384:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574008.450386:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574008.450395:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574008.450399:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574008.450403:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574008.450407:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574008.450412:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7480 x1875440516643840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574008.450425:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7480 x1875440516643840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450434:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574008.450437:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.450440:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574008.450443:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574008.450447:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7480 x1875440516643840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450456:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574008.450461:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7480 x1875440516643840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450470:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574008.450475:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.450479:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574008.450481:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.450485:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574008.450488:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574008.450492:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574008.450499:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574008.450504:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574008.450510:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7480 x1875440516643840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574008.450544:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516643840:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574008.450555:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574008.450559:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6680 x1875440516643968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450564:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574008.450566:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574008.450567:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574008.450569:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574008.450571:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574008.450573:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516643968:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574008.450576:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574008.450577:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574008.450578:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574008.450580:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574008.450582:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574008.450583:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574008.450586:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574008.450587:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574008.450590:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574008.450592:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516643968, portal 10 00000100:00000001:0.0:1788574008.450593:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574008.450594:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605330560 : -123400104221056 : ffff8fc4ac9f6680) 00000100:00000040:0.0:1788574008.450598:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6680 x1875440516643968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450602:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574008.450604:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574008.450605:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574008.450607:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516643968, offset 0 00000400:00000200:0.0:1788574008.450609:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574008.450628:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574008.450631:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574008.450633:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574008.450639:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574008.450641:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574008.450643:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574008.450645:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546683:546683:256:4294967295] 192.168.202.128@tcp LPNI seq info [546606:546606:8:4294967295] 00000400:00000200:0.0:1788574008.450648:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574008.450652:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574008.450656:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574008.450658:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574008.450661:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574008.450662:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788574008.450669:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574008.450671:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574008.450673:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574008.450675:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574008.450677:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574008.450678:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574008.450680:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574008.450683:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6680 x1875440516643968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450688:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574008.450690:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6680 x1875440516643968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450695:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.450696:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574008.450698:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574008.450700:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574008.450702:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574008.450703:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309368346 00000100:00000001:0.0:1788574008.450705:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309372141 00000100:00000001:0.0:1788574008.450707:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309368346 00000100:00000001:0.0:1788574008.450712:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309372141 00000100:00000001:0.0:1788574008.450713:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574008.450715:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.450716:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574008.450719:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574008/real 1788574008] req@ffff8fc4ac9f6680 x1875440516643968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574008.450724:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574008.450725:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574008.450727:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574008.450728:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574008.450729:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574008.450731:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574008.450733:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6680 x1875440516643968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574008.450738:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6680 x1875440516643968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450741:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574008.450742:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.450743:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574008.450744:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574008.450746:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6680 x1875440516643968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450750:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574008.450753:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6680 x1875440516643968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450757:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574008.450759:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.450760:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574008.450761:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.450765:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574008.450767:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574008.450768:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574008.450771:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574008.450772:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574008.450775:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6680 x1875440516643968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574008.450780:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516643968:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574008.450782:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.450784:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574008.450786:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7480 x1875440516643840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450789:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574008.450790:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574008.450792:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574008.450793:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574008.450794:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574008.450795:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574008.450797:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574008.450798:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36400. 02000000:00000001:0.0:1788574008.450800:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574008.450801:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574008.450802:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7480. 00000100:00000001:0.0:1788574008.450804:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574008.450804:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.450805:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574008.450821:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6680 x1875440516643968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450827:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574008.450828:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574008.450829:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574008.450835:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574008.450836:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574008.450837:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574008.450838:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574008.450839:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35400. 02000000:00000001:0.0:1788574008.450841:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574008.450842:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574008.450843:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6680. 00000100:00000001:0.0:1788574008.450845:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574008.450845:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.450847:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.450849:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574008.450850:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.450852:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574008.450853:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574008.450853:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574008.450856:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7100 x1875440516644096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450861:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574008.450862:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574008.450864:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574008.450865:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574008.450866:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574008.450868:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516644096:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574008.450870:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574008.450872:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574008.450873:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574008.450874:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574008.450876:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574008.450877:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574008.450880:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574008.450882:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574008.450885:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574008.450887:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516644096, portal 4 00000100:00000001:0.0:1788574008.450888:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574008.450892:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605333248 : -123400104218368 : ffff8fc4ac9f7100) 00000100:00000040:0.0:1788574008.450896:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7100 x1875440516644096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450900:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574008.450902:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574008.450903:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574008.450905:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516644096, offset 0 00000400:00000200:0.0:1788574008.450907:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574008.450910:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574008.450912:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574008.450914:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574008.450916:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574008.450917:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574008.450919:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574008.450921:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546684:546684:256:4294967295] 192.168.202.128@tcp LPNI seq info [546607:546607:8:4294967295] 00000400:00000200:0.0:1788574008.450924:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574008.450928:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574008.450931:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574008.450933:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574008.450935:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574008.450937:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788574008.450944:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574008.450946:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574008.450948:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574008.450950:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574008.450952:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574008.450953:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574008.450955:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574008.450957:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7100 x1875440516644096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450962:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574008.450967:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7100 x1875440516644096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.450972:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.450973:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574008.450975:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574008.450977:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574008.450979:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574008.450981:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309368487 00000100:00000001:0.0:1788574008.450983:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309372141 00000100:00000001:0.0:1788574008.450984:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309368487 00000100:00000001:0.0:1788574008.450986:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309372141 00000100:00000001:0.0:1788574008.450987:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574008.450988:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.450990:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574008.450992:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574008/real 1788574008] req@ffff8fc4ac9f7100 x1875440516644096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574008.450997:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574008.450997:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574008.450998:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574008.451000:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574008.451001:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574008.451002:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574008.451004:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7100 x1875440516644096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574008.451009:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7100 x1875440516644096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.451013:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574008.451014:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.451015:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574008.451016:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574008.451023:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7100 x1875440516644096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.451027:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574008.451030:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7100 x1875440516644096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.451034:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574008.451035:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.451036:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574008.451037:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.451038:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574008.451040:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574008.451041:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574008.451043:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574008.451044:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574008.451047:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7100 x1875440516644096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574008.451051:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516644096:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574008.451054:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574008.451056:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4000 x1875440516644224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.451060:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574008.451061:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574008.451062:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574008.451063:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574008.451064:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574008.451066:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516644224:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574008.451068:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574008.451072:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574008.451073:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574008.451074:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574008.451076:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574008.451077:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574008.451079:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574008.451081:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574008.451083:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574008.451085:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516644224, portal 4 00000100:00000001:0.0:1788574008.451086:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574008.451087:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605320704 : -123400104230912 : ffff8fc4ac9f4000) 00000100:00000040:0.0:1788574008.451091:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4000 x1875440516644224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.451095:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574008.451096:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574008.451098:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574008.451099:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516644224, offset 0 00000400:00000200:0.0:1788574008.451101:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574008.451104:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574008.451106:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574008.451107:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574008.451109:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574008.451111:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574008.451112:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574008.451114:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [546685:546685:256:4294967295] 192.168.202.128@tcp LPNI seq info [546608:546608:8:4294967295] 00000400:00000200:0.0:1788574008.451117:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574008.451121:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574008.451123:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574008.451125:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574008.451128:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574008.451129:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788574008.451140:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574008.451143:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574008.451145:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574008.451148:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574008.451150:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574008.451151:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574008.451153:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574008.451155:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4000 x1875440516644224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.451160:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574008.451162:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4000 x1875440516644224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574033 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.451166:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.451167:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574008.451169:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574008.451171:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574008.451173:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574008.451174:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309368489 00000100:00000001:0.0:1788574008.451176:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309372141 00000100:00000001:0.0:1788574008.451177:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309368489 00000100:00000001:0.0:1788574008.451179:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309372141 00000100:00000001:0.0:1788574008.451180:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574008.451182:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.451183:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574008.451185:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574008/real 1788574008] req@ffff8fc4ac9f4000 x1875440516644224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574008.451190:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574008.451190:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574008.451191:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574008.451193:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574008.451194:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574008.451198:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574008.451201:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4000 x1875440516644224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574008.451206:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4000 x1875440516644224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.451210:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574008.451211:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.451212:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574008.451213:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574008.451215:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4000 x1875440516644224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.451219:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574008.451221:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4000 x1875440516644224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.451226:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574008.451227:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.451228:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574008.451229:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.451230:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574008.451231:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574008.451233:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574008.451235:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574008.451236:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574008.451238:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4000 x1875440516644224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574008.451243:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516644224:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574008.451245:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.451246:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574008.451251:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7100 x1875440516644096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.451255:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574008.451256:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574008.451258:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574008.451259:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574008.451260:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574008.451261:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574008.451263:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574008.451264:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35c00. 02000000:00000001:0.0:1788574008.451265:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574008.451266:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574008.451267:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7100. 00000100:00000001:0.0:1788574008.451269:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574008.451270:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.451271:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574008.451273:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4000 x1875440516644224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574033 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574008.451277:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574008.451278:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574008.451279:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574008.451280:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574008.451281:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574008.451281:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574008.451282:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574008.451283:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788574008.451285:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574008.451285:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574008.451286:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4000. 00000100:00000001:0.0:1788574008.451287:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574008.451288:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.451289:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.451292:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574008.451293:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.451294:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574008.451295:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574008.960903:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574008.960913:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574008.960917:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574008.960927:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574008.960935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574008.960941:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.960961:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574008.960965:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574008.960971:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574009.664998:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574009.665001:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574009.665004:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574009.984891:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574009.984898:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574009.984900:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574009.984909:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574009.984916:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574009.984921:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574009.984928:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574009.984933:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574009.984938:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574009.984962:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574009.984968:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574009.984971:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574009.984975:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574009.984979:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574009.984983:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574009.984988:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574009.984990:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574009.984992:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574011.008920:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574011.008928:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574011.008931:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574011.008936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574011.008942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574011.008946:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574011.008950:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574011.008954:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574011.009162:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574011.009188:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574011.009191:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574011.009193:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574011.009195:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574011.009197:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574011.009199:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574011.009201:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574011.009203:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574011.009204:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574012.032914:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574012.032923:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574012.032929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574012.032936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574012.032944:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574012.032958:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574012.032989:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574012.032993:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574012.032997:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574012.288884:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309372141, at 4309372160 00000100:00000010:0.0:1788574012.288902:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788574012.737047:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574012.737053:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574012.737056:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574013.056959:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574013.056973:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574013.056986:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574013.057006:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574013.057025:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574013.057043:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574013.057059:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574013.057062:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574013.057065:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574013.568963:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574013.568974:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574013.568977:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574013.569006:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574013.569011:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574013.569019:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14701 00000400:00000200:0.0:1788574013.569029:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574013.569035:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574013.569038:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574013.569041:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569044:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574013.569047:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569049:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574013.569053:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574013.569055:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574013.569057:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574013.569060:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574013.569062:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574013.569064:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574013.569066:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569067:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574013.569068:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569070:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574013.569073:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574013.569076:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569077:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574013.569080:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574013.569083:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574013.569085:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.569087:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574013.569089:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574013.569096:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574013.569099:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574013.569102:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574013.569108:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.569112:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574013.569115:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574013.569120:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574013.569129:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574013.569136:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574013.569145:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516644352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574013.569154:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574013.569159:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574013.569165:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516644352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574013.569196:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574013.569200:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574013.569205:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574013.569206:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574013.569208:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574013.569211:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574013.569215:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14701 00000400:00000200:0.0:1788574013.569219:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574013.569222:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574013.569224:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574013.569227:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569232:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574013.569234:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569239:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574013.569243:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574013.569244:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574013.569247:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574013.569250:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574013.569255:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574013.569264:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574013.569272:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569276:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574013.569279:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569283:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574013.569287:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574013.569293:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569296:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574013.569299:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574013.569305:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574013.569308:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.569311:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574013.569316:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574013.569319:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.569321:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574013.569325:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574013.569329:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574013.569333:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574013.569334:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.569335:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574013.569337:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574013.569340:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574013.569348:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574013.569351:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574013.569357:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516644480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574013.569367:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574013.569370:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574013.569374:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516644480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574013.569388:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574013.569397:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574013.569401:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574013.569402:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574013.569403:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574013.569408:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574013.569413:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14701 00000400:00000200:0.0:1788574013.569419:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574013.569425:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574013.569429:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574013.569434:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569437:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574013.569440:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569443:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574013.569446:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574013.569449:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574013.569450:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574013.569451:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574013.569453:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574013.569454:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574013.569461:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569464:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574013.569469:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569474:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574013.569477:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574013.569481:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569484:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574013.569487:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574013.569491:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574013.569495:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.569498:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574013.569507:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574013.569511:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.569512:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574013.569515:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574013.569520:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574013.569525:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574013.569527:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.569531:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574013.569534:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574013.569536:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574013.569541:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574013.569546:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574013.569553:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516644608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574013.569565:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574013.569569:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574013.569575:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516644608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574013.569589:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574013.569591:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574013.569598:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574013.569601:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574013.569603:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574013.569608:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574013.569611:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14701 00000400:00000200:0.0:1788574013.569618:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574013.569623:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574013.569626:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574013.569646:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569652:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574013.569658:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569669:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574013.569675:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574013.569677:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574013.569679:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574013.569681:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574013.569686:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574013.569691:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574013.569699:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569703:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574013.569707:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569708:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574013.569711:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574013.569717:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569721:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574013.569725:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574013.569729:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574013.569732:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.569735:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574013.569737:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574013.569739:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.569741:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574013.569746:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574013.569749:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574013.569753:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574013.569755:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.569759:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574013.569760:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574013.569772:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574013.569779:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574013.569786:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574013.569793:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516644736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574013.569865:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574013.569869:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574013.569878:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516644736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574013.569892:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574013.569895:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574013.569899:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574013.569901:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574013.569902:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574013.569907:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574013.569912:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14701 00000400:00000200:0.0:1788574013.569919:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574013.569925:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574013.569929:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574013.569934:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569939:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574013.569942:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569946:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574013.569951:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574013.569954:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574013.569957:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574013.569961:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574013.569966:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574013.569972:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574013.569978:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569981:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574013.569984:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574013.569985:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574013.569995:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574013.570003:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574013.570008:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574013.570011:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574013.570016:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574013.570019:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.570022:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574013.570026:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574013.570033:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.570038:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574013.570042:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574013.570046:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574013.570050:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574013.570053:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574013.570055:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574013.570059:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574013.570062:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574013.570070:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574013.570076:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574013.570082:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516644864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574013.570093:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574013.570097:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574013.570103:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516644864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574013.570111:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574013.570117:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14711) 00000100:00000001:0.0:1788574014.080935:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574014.080940:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574014.080944:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574014.080950:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574014.080957:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574014.080972:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574014.080979:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574014.080984:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574014.080989:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574014.784985:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574014.784990:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574014.784994:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574015.104944:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574015.104949:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574015.104953:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574015.104957:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574015.104960:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574015.104963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574015.104966:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574015.104968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574015.104970:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574016.128909:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574016.128917:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574016.128919:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574016.128924:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574016.128927:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574016.128932:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574016.128936:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574016.128940:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574016.128942:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574016.128977:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574016.128983:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574016.128986:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574016.128993:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574016.129000:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574016.129006:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574016.129013:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574016.129016:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574016.129020:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574017.152930:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574017.152938:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574017.152944:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574017.152964:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574017.152989:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574017.152993:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574017.152997:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574017.152999:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574017.153002:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574017.408914:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309377261, at 4309377280 00000100:00000010:0.0:1788574017.408952:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea73c0. 00000020:00000001:0.0:1788574017.857084:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574017.857087:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574017.857090:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574018.176900:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574018.176906:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574018.176910:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574018.176916:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574018.176922:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574018.176928:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574018.176934:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574018.176939:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574018.176946:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574019.200905:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574019.200912:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574019.200916:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574019.200921:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574019.200930:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574019.200937:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574019.200957:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574019.200961:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574019.200969:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574019.905015:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574019.905018:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574019.905020:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574020.224893:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574020.224898:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574020.224903:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574020.224908:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574020.224914:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574020.224919:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574020.224949:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574020.224953:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574020.224958:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574021.248924:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574021.248932:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574021.248937:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574021.248945:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574021.248951:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574021.248956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574021.248962:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574021.248965:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574021.248972:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574022.272954:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574022.272960:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574022.272961:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574022.272965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574022.272968:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574022.272971:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574022.272973:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574022.272975:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574022.272977:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574022.528884:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309382381, at 4309382400 00000100:00000010:0.0:1788574022.528901:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7a80. 00000020:00000001:0.0:1788574022.976956:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574022.976960:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574022.976963:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574023.296886:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574023.296894:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574023.296897:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574023.296904:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574023.296912:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574023.296919:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574023.296928:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574023.296932:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574023.296937:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574024.320938:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574024.320942:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574024.320954:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574024.320958:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574024.320963:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574024.320969:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574024.320987:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574024.320990:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574024.320993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574025.024962:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574025.024965:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574025.024968:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574025.344983:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574025.344990:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574025.344995:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574025.345002:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574025.345005:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574025.345008:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574025.345021:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574025.345026:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574025.345031:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574027.392899:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574027.392905:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574027.392909:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574027.392914:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574027.392922:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574027.392925:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574027.392944:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574027.392947:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574027.392951:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574027.648918:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309387501, at 4309387520 00000100:00000010:0.0:1788574027.648936:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7420. 00000020:00000001:0.0:1788574028.097007:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574028.097010:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574028.097012:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574028.416923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574028.416931:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574028.416936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574028.416946:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574028.416966:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574028.416971:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574028.416997:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574028.417001:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574028.417004:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574029.441015:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574029.441021:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574029.441024:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574029.441026:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574029.441028:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574029.441030:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574029.441315:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574029.441317:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574029.441319:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574030.144965:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574030.144968:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574030.144970:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574030.464911:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574030.464921:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574030.464925:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574030.464934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574030.464941:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574030.464946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574030.464960:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574030.464965:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574030.464970:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574031.488938:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574031.488943:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574031.488946:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574031.488952:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574031.488956:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574031.488960:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574031.488972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574031.488975:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574031.488979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574032.513044:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574032.513054:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574032.513058:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574032.513071:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574032.513076:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574032.513080:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574032.513082:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574032.513084:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574032.513086:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574032.768921:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309392621, at 4309392640 00000100:00000010:0.0:1788574032.768937:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ba0. 00000020:00000001:0.0:1788574033.216989:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574033.216993:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574033.216996:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574033.536926:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574033.536935:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574033.536940:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574033.536950:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574033.536957:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574033.536964:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574033.536984:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574033.536987:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574033.536991:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574033.537025:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574033.537030:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574033.537034:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574033.537040:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574033.537045:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574033.537050:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574033.537053:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574033.537055:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574033.537059:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574034.049152:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788574035.265054:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574035.265057:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574035.265060:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574035.584945:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574035.584950:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574035.584954:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574035.584961:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574035.584982:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574035.584987:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574035.584991:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574035.584994:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574035.584998:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574036.608961:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574036.608973:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574036.608985:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574036.608991:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574036.609001:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574036.609011:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574036.609028:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574036.609036:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574036.609042:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788574037.121050:0:2388:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788574037.121094:0:2388:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788574037.121104:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574037.121108:0:2388:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788574037.121113:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574037.121117:0:2388:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788574037.121123:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574037.121127:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc488504900. 00000400:00000200:0.0:1788574037.121132:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574037.121137:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574037.121144:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed110 00000400:00000010:0.0:1788574037.121146:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed110. 00000100:00000001:0.0:1788574037.121153:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574037.121162:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a4a80 x1875440516646912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574037.121192:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574037.121196:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516646912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574059 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574037.121202:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574037.121227:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574037.121240:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574037.121244:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc488504f00. 00000400:00000200:0.0:1788574037.121248:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574037.121256:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574037.121262:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76edcc0 00000400:00000010:0.0:1788574037.121264:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76edcc0. 00000100:00000001:0.0:1788574037.121269:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574037.121276:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a7800 x1875440516647040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574037.121289:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574037.121297:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516647040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574059 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574037.121307:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574037.121313:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574037.121322:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574037.121327:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc488504b00. 00000400:00000200:0.0:1788574037.121335:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574037.121345:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574037.121351:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed198 00000400:00000010:0.0:1788574037.121356:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed198. 00000100:00000001:0.0:1788574037.121363:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574037.121373:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a5180 x1875440516647168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574037.121391:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574037.121398:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516647168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574059 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574037.121412:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574037.121414:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574037.121423:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574037.121430:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc488504700. 00000400:00000200:0.0:1788574037.121444:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574037.121462:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574037.121467:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed880 00000400:00000010:0.0:1788574037.121472:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed880. 00000100:00000001:0.0:1788574037.121476:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574037.121483:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a5c00 x1875440516647296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574037.121498:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574037.121504:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516647296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574059 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574037.121516:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574037.121522:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574037.121531:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574037.121537:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc488504c00. 00000400:00000200:0.0:1788574037.121543:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574037.121553:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574037.121559:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed000 00000400:00000010:0.0:1788574037.121565:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed000. 00000100:00000001:0.0:1788574037.121570:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574037.121579:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4901a7100 x1875440516647424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574037.121615:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574037.121623:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516647424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574059 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574037.121635:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574037.121640:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574037.633068:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574037.633073:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574037.633075:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574037.633078:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574037.633081:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574037.633083:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574037.633086:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574037.633097:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574037.633100:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574037.888900:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309397741, at 4309397760 00000100:00000010:0.0:1788574037.888914:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abb40. 00000020:00000001:0.0:1788574038.337052:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574038.337058:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574038.337062:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574038.656967:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574038.656985:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574038.656996:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574038.657011:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574038.657025:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574038.657031:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574038.657039:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574038.657044:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574038.657050:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574038.657072:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574038.657076:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574038.657081:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574038.657084:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574038.657086:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574038.657089:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574038.657107:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574038.657111:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574038.657115:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574039.680909:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574039.680919:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574039.680923:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574039.680942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574039.680953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574039.680959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574039.680974:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574039.680977:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574039.680981:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574040.385010:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574040.385015:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574040.385018:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574040.705013:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574040.705023:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574040.705033:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574040.705047:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574040.705057:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574040.705061:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574040.705065:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574040.705069:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574040.705072:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574041.728875:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574041.728883:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574041.728885:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574041.728890:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574041.728895:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574041.728900:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574041.728904:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574041.728915:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574041.728921:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574042.752891:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574042.752898:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574042.752900:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574042.752906:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574042.752910:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574042.752915:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574042.752919:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574042.752923:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574042.752929:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574043.008922:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309402862, at 4309402880 00000100:00000010:0.0:1788574043.008939:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab720. 00000020:00000001:0.0:1788574043.457003:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574043.457007:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574043.457011:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574043.776985:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574043.776992:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574043.776996:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574043.777000:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574043.777006:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574043.777013:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574043.777016:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574043.777018:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574043.777021:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574044.800908:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574044.800913:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574044.800919:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574044.800927:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574044.800933:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574044.800940:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574044.800961:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574044.800963:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574044.800968:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574045.505011:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574045.505014:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574045.505017:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574045.824914:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574045.824923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574045.824928:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574045.824935:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574045.824941:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574045.824945:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574045.824973:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574045.824977:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574045.824979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574046.848929:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574046.848936:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574046.848941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574046.848948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574046.848954:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574046.848958:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574046.848988:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574046.848991:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574046.848995:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574047.872921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574047.872930:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574047.872935:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574047.872942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574047.872965:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574047.872970:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574047.872978:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574047.872983:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574047.872990:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574048.128924:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309407981, at 4309408000 00000100:00000010:0.0:1788574048.128941:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab540. 00000020:00000001:0.0:1788574048.577011:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574048.577015:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574048.577018:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574048.896885:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574048.896893:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574048.896896:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574048.896904:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574048.896910:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574048.896914:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574048.896917:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574048.896921:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574048.896926:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574048.896957:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574048.896961:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574048.896963:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574048.896966:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574048.896969:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574048.896972:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574048.896976:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574048.896979:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574048.896981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574049.920923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574049.920933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574049.920937:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574049.920945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574049.920952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574049.920958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574049.920966:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574049.920969:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574049.920972:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574050.625030:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574050.625033:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574050.625035:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574050.944907:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574050.944915:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574050.944918:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574050.944926:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574050.944932:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574050.944936:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574050.945178:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574050.945181:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574050.945184:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574050.945214:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574050.945217:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574050.945218:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574050.945221:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574050.945224:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574050.945226:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574050.945228:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574050.945230:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574050.945232:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574051.968919:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574051.968925:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574051.968929:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574051.968935:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574051.968943:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574051.968948:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574051.968955:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574051.968961:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574051.968968:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574052.992915:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574052.992923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574052.992927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574052.992934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574052.992941:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574052.992954:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574052.992985:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574052.992988:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574052.993009:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574053.248880:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309413101, at 4309413120 00000100:00000010:0.0:1788574053.248898:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abb40. 00000020:00000001:0.0:1788574053.696967:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574053.696971:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574053.696975:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574054.016956:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574054.016960:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574054.016963:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574054.016968:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574054.016974:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574054.016979:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574054.016991:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574054.016993:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574054.017003:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574055.040902:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574055.040911:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574055.040914:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574055.040921:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574055.040928:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574055.040936:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574055.040942:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574055.040947:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574055.040951:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574055.040970:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574055.040975:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574055.040978:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574055.040980:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574055.040983:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574055.040985:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574055.040987:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574055.040988:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574055.040990:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574055.744984:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574055.744989:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574055.744993:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574056.064912:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574056.064933:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574056.064943:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574056.064948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574056.064953:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574056.064958:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574056.064991:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574056.064995:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574056.065000:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574057.088885:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574057.088890:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574057.088893:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574057.088898:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574057.088906:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574057.088912:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574057.088918:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574057.088922:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574057.088929:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574057.088966:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574057.088969:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574057.088970:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574057.088972:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574057.088975:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574057.088978:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574057.088980:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574057.088982:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574057.088983:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574058.112936:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574058.112944:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574058.112954:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574058.112959:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574058.112966:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574058.112971:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574058.112977:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574058.112979:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574058.112982:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574058.113025:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574058.113028:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574058.113029:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574058.113041:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574058.113045:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574058.113048:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574058.113052:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574058.113054:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574058.113060:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574058.368908:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309418221, at 4309418240 00000100:00000010:0.0:1788574058.368931:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7b40. 00000020:00000001:0.0:1788574058.817000:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574058.817003:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574058.817006:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574059.136909:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574059.136915:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574059.136917:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574059.136925:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574059.136931:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574059.136935:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574059.136940:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574059.136945:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574059.136950:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574059.136980:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574059.136985:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574059.136990:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574059.136997:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574059.137001:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574059.137003:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574059.137006:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574059.137008:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574059.137010:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574060.160913:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574060.160919:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574060.160922:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574060.160928:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574060.160935:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574060.160941:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574060.160958:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574060.160962:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574060.160974:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574060.865003:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574060.865007:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574060.865010:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574061.184922:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574061.184927:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574061.184931:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574061.184939:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574061.184948:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574061.184952:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574061.184965:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574061.184969:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574061.184972:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574062.208921:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574062.208929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574062.208934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574062.208940:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574062.208947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574062.208954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574062.208958:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574062.208961:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574062.208965:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574063.232919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574063.232929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574063.232931:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574063.232938:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574063.232944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574063.232949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574063.232954:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574063.232958:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574063.232963:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574063.488909:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309423341, at 4309423360 00000100:00000010:0.0:1788574063.488929:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7360. 00000020:00000001:0.0:1788574063.937023:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574063.937028:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574063.937032:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574064.256965:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574064.256987:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574064.256990:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574064.256997:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574064.257003:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574064.257009:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574064.257016:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574064.257021:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574064.257029:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574065.280920:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574065.280929:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574065.280933:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574065.280940:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574065.280945:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574065.280949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574065.280961:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574065.280964:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574065.280967:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574065.985030:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574065.985035:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574065.985039:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574066.304946:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574066.304952:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574066.304955:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574066.304959:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574066.304964:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574066.304969:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574066.304971:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574066.304973:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574066.304975:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574067.328951:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574067.328957:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574067.328959:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574067.328963:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574067.328969:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574067.328975:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574067.328981:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574067.328984:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574067.328987:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574068.352903:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574068.352910:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574068.352914:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574068.352922:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574068.352929:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574068.352934:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574068.352953:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574068.352955:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574068.352964:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574068.608889:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309428461, at 4309428480 00000100:00000010:0.0:1788574068.608911:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ea0. 00000020:00000001:0.0:1788574069.056982:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574069.056987:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574069.056990:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574069.376926:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574069.376935:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574069.376940:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574069.376946:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574069.376951:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574069.376956:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574069.376988:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574069.376992:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574069.376996:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574069.888904:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574069.888917:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574069.888920:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574069.888929:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574069.888933:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574069.888939:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14757 00000400:00000200:0.0:1788574069.888947:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574069.888952:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574069.888954:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574069.888957:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.888959:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574069.888975:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.888977:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574069.888980:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574069.888982:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574069.888984:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574069.888986:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574069.888988:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574069.888990:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574069.888992:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.888993:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574069.888994:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.888995:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574069.888997:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574069.889000:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889002:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574069.889003:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574069.889006:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574069.889008:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.889009:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574069.889011:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574069.889018:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574069.889021:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574069.889025:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574069.889027:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.889030:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574069.889032:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574069.889037:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574069.889045:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574069.889052:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574069.889062:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516651392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574069.889073:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574069.889084:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574069.889091:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516651392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574069.889114:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574069.889118:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574069.889121:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574069.889123:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574069.889124:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574069.889127:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574069.889130:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14757 00000400:00000200:0.0:1788574069.889135:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574069.889148:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574069.889154:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574069.889160:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889164:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574069.889170:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889174:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574069.889177:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574069.889180:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574069.889182:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574069.889186:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574069.889188:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574069.889192:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574069.889195:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889199:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574069.889207:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889211:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574069.889216:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574069.889224:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889238:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574069.889244:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574069.889252:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574069.889258:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.889262:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574069.889269:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574069.889275:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.889278:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574069.889284:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574069.889288:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574069.889293:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574069.889295:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.889297:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574069.889298:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574069.889300:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574069.889306:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574069.889321:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574069.889329:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516651520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574069.889338:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574069.889342:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574069.889348:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516651520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574069.889360:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574069.889369:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574069.889374:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574069.889378:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574069.889381:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574069.889386:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574069.889391:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14757 00000400:00000200:0.0:1788574069.889407:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574069.889425:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574069.889428:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574069.889436:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889440:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574069.889445:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889448:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574069.889451:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574069.889454:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574069.889456:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574069.889458:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574069.889463:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574069.889467:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574069.889473:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889491:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574069.889492:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889494:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574069.889496:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574069.889501:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889506:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574069.889508:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574069.889514:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574069.889517:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.889525:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574069.889528:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574069.889532:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.889536:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574069.889541:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574069.889544:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574069.889546:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574069.889548:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.889552:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574069.889560:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574069.889563:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574069.889567:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574069.889573:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574069.889580:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516651648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574069.889590:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574069.889594:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574069.889600:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516651648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574069.889613:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574069.889618:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574069.889631:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574069.889634:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574069.889637:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574069.889640:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574069.889644:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14757 00000400:00000200:0.0:1788574069.889652:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574069.889658:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574069.889661:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574069.889667:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889672:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574069.889676:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889680:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574069.889683:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574069.889685:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574069.889688:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574069.889691:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574069.889694:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574069.889698:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574069.889709:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889713:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574069.889715:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889717:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574069.889719:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574069.889725:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889730:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574069.889733:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574069.889738:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574069.889742:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.889744:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574069.889747:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574069.889752:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.889756:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574069.889760:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574069.889764:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574069.889769:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574069.889773:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.889777:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574069.889779:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574069.889782:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574069.889788:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574069.889794:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574069.889802:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516651776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574069.889857:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574069.889863:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574069.889873:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516651776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574069.889888:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574069.889891:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574069.889902:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574069.889905:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574069.889908:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574069.889911:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574069.889914:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14757 00000400:00000200:0.0:1788574069.889921:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574069.889926:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574069.889929:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574069.889934:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889938:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574069.889942:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889946:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574069.889949:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574069.889952:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574069.889956:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574069.889961:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574069.889967:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574069.889972:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574069.889978:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889981:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574069.889984:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889989:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574069.889993:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574069.889995:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574069.889999:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574069.890002:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574069.890007:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574069.890010:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.890011:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574069.890014:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574069.890031:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.890033:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574069.890038:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574069.890043:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574069.890048:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574069.890049:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574069.890052:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574069.890054:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574069.890057:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574069.890062:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574069.890067:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574069.890074:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516651904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574069.890084:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574069.890087:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574069.890093:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516651904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574069.890105:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574069.890111:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14767) 00000100:00000001:0.0:1788574070.400957:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574070.400962:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574070.400966:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574070.400973:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574070.400979:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574070.400985:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574070.400989:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574070.400993:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574070.400999:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574071.104999:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574071.105004:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574071.105023:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574071.424948:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574071.424954:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574071.424957:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574071.424972:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574071.424978:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574071.424983:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574071.424996:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574071.425001:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574071.425008:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574072.961097:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788574073.472933:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574073.472940:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574073.472942:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574073.472951:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574073.472957:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574073.472962:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574073.472981:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574073.472987:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574073.472992:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574073.728935:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309433581, at 4309433600 00000100:00000010:0.0:1788574073.728959:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f00. 00000020:00000001:0.0:1788574074.177041:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574074.177044:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574074.177047:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574074.497036:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574074.497039:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574074.497040:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574074.497043:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574074.497048:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574074.497049:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574074.497260:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574074.497263:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574074.497265:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574075.008894:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574075.008904:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574075.008907:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574075.008911:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574075.008916:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574075.008923:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14762 00000400:00000200:0.0:1788574075.008947:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574075.008953:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574075.008955:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574075.008958:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.008960:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574075.008962:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.008964:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574075.008967:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574075.008968:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574075.008970:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574075.008972:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574075.008974:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574075.008976:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574075.008978:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.008979:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574075.008981:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.008982:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574075.008984:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574075.008986:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574075.008988:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574075.008990:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574075.008992:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574075.008994:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.008996:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574075.008997:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574075.009002:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574075.009005:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574075.009008:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574075.009009:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.009011:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574075.009014:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574075.009017:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574075.009027:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574075.009032:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574075.009040:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516652032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574075.009048:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574075.009051:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574075.009055:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516652032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574075.009074:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574075.009076:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574075.009079:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574075.009080:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574075.009081:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574075.009083:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574075.009086:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14762 00000400:00000200:0.0:1788574075.009090:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574075.009093:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574075.009096:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574075.009100:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009113:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574075.009116:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009118:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574075.009121:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574075.009125:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574075.009128:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574075.009133:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574075.009139:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574075.009145:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574075.009152:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009161:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574075.009165:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009179:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574075.009183:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574075.009194:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009196:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574075.009200:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574075.009205:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574075.009210:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.009212:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574075.009216:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574075.009221:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.009227:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574075.009231:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574075.009237:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574075.009244:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574075.009247:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.009251:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574075.009254:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574075.009259:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574075.009268:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574075.009275:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574075.009287:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516652160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574075.009304:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574075.009309:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574075.009319:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516652160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574075.009329:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574075.009333:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574075.009338:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574075.009348:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574075.009351:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574075.009358:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574075.009365:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14762 00000400:00000200:0.0:1788574075.009383:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574075.009389:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574075.009392:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574075.009395:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009400:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574075.009402:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009404:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574075.009406:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574075.009408:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574075.009410:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574075.009413:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574075.009416:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574075.009419:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574075.009426:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009430:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574075.009463:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009471:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574075.009475:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574075.009483:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009488:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574075.009492:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574075.009497:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574075.009504:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.009508:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574075.009512:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574075.009517:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.009528:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574075.009536:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574075.009543:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574075.009548:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574075.009552:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.009556:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574075.009559:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574075.009562:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574075.009568:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574075.009573:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574075.009578:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516652288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574075.009587:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574075.009591:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574075.009596:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516652288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574075.009611:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574075.009616:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574075.009621:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574075.009625:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574075.009627:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574075.009632:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574075.009637:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14762 00000400:00000200:0.0:1788574075.009643:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574075.009648:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574075.009652:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574075.009657:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009660:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574075.009665:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009669:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574075.009674:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574075.009681:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574075.009685:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574075.009688:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574075.009692:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574075.009696:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574075.009703:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009707:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574075.009711:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009716:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574075.009720:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574075.009726:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009732:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574075.009736:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574075.009740:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574075.009744:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.009748:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574075.009751:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574075.009755:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.009758:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574075.009762:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574075.009763:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574075.009765:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574075.009766:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.009767:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574075.009768:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574075.009769:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574075.009772:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574075.009778:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574075.009789:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516652416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574075.009839:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574075.009843:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574075.009858:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516652416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574075.009874:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574075.009878:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574075.009882:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574075.009885:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574075.009888:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574075.009892:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574075.009897:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14762 00000400:00000200:0.0:1788574075.009901:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574075.009905:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574075.009910:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574075.009915:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009921:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574075.009927:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009932:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574075.009938:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574075.009941:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574075.009946:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574075.009951:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574075.009958:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574075.009965:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574075.009975:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009982:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574075.009987:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574075.009991:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574075.009995:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574075.010001:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574075.010004:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574075.010014:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574075.010017:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574075.010019:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.010020:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574075.010025:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574075.010030:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.010035:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574075.010043:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574075.010049:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574075.010056:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574075.010060:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574075.010065:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574075.010068:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574075.010072:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574075.010080:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574075.010088:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574075.010099:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516652544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574075.010112:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574075.010117:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574075.010127:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516652544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574075.010148:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574075.010153:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14772) 00000020:00000001:0.0:1788574076.225004:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574076.225012:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574076.225017:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574076.544997:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574076.545007:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574076.545010:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574076.545015:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574076.545020:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574076.545026:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574076.545029:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574076.545050:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574076.545052:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574077.568986:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574077.568997:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574077.569006:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574077.569016:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574077.569030:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574077.569038:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574077.569056:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574077.569062:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574077.569068:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574078.592887:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574078.592895:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574078.592898:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574078.592908:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574078.592916:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574078.592923:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574078.592948:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574078.592952:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574078.592956:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574078.848902:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309438701, at 4309438720 00000100:00000010:0.0:1788574078.848922:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7de0. 00000020:00000001:0.0:1788574079.296969:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574079.296973:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574079.296975:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574079.616919:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574079.616925:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574079.616928:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574079.616934:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574079.616940:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574079.616944:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574079.616949:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574079.616953:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574079.616958:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574080.640892:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574080.640899:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574080.640901:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574080.640926:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574080.640932:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574080.640937:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574080.640940:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574080.640944:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574080.640947:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574080.640970:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574080.640974:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574080.640976:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574080.640981:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574080.640983:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574080.640987:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574080.640992:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574080.640994:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574080.641000:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574081.344987:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574081.344992:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574081.344996:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574081.664902:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574081.664911:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574081.664916:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574081.664926:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574081.664932:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574081.664943:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574081.664969:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574081.664972:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574081.664974:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574082.688926:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574082.688934:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574082.688937:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574082.688945:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574082.688952:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574082.688965:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574082.688972:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574082.688976:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574082.688982:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574083.968925:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309443821, at 4309443840 00000100:00000010:0.0:1788574083.968941:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7840. 00000020:00000001:0.0:1788574084.417085:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574084.417091:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574084.417095:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574084.737015:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574084.737023:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574084.737031:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574084.737040:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574084.737058:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574084.737070:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574084.737075:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574084.737078:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574084.737082:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574085.248931:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574085.248942:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574085.248945:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574085.248948:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574085.248953:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574085.248960:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14772 00000400:00000200:0.0:1788574085.248968:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574085.248974:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574085.248977:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574085.248979:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.248983:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574085.248985:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.248987:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574085.248990:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574085.248991:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574085.248993:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574085.248996:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574085.248997:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574085.248999:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574085.249001:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249003:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574085.249021:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249023:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574085.249025:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574085.249028:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249030:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574085.249032:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574085.249034:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574085.249036:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249038:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574085.249039:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574085.249044:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574085.249047:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574085.249050:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574085.249052:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249054:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574085.249056:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574085.249060:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574085.249066:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574085.249072:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574085.249079:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516653312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574085.249086:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574085.249090:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574085.249094:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516653312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574085.249110:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574085.249114:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574085.249117:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574085.249119:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574085.249121:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574085.249124:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574085.249134:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14772 00000400:00000200:0.0:1788574085.249142:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574085.249146:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574085.249149:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574085.249157:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249162:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574085.249168:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249172:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574085.249177:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574085.249179:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574085.249181:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574085.249183:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574085.249187:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574085.249190:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574085.249197:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249199:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574085.249212:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249215:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574085.249217:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574085.249223:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249226:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574085.249228:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574085.249234:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574085.249240:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249243:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574085.249246:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574085.249250:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249251:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574085.249255:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574085.249260:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574085.249265:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574085.249273:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249278:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574085.249280:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574085.249285:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574085.249291:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574085.249295:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574085.249299:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516653440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574085.249310:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574085.249312:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574085.249318:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516653440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574085.249329:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574085.249334:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574085.249340:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574085.249341:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574085.249343:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574085.249346:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574085.249350:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14772 00000400:00000200:0.0:1788574085.249369:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574085.249374:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574085.249378:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574085.249384:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249388:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574085.249392:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249395:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574085.249398:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574085.249401:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574085.249405:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574085.249409:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574085.249415:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574085.249427:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574085.249437:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249443:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574085.249449:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249454:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574085.249456:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574085.249461:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249467:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574085.249471:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574085.249475:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574085.249480:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249484:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574085.249486:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574085.249489:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249490:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574085.249494:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574085.249499:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574085.249504:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574085.249506:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249510:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574085.249512:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574085.249515:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574085.249519:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574085.249525:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574085.249534:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516653568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574085.249543:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574085.249546:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574085.249553:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516653568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574085.249567:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574085.249575:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574085.249578:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574085.249583:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574085.249584:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574085.249589:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574085.249594:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14772 00000400:00000200:0.0:1788574085.249600:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574085.249603:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574085.249605:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574085.249610:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249612:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574085.249614:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249616:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574085.249620:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574085.249621:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574085.249622:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574085.249624:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574085.249630:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574085.249636:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574085.249642:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249647:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574085.249652:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249658:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574085.249664:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574085.249672:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249677:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574085.249682:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574085.249687:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574085.249692:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249695:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574085.249705:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574085.249711:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249717:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574085.249725:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574085.249732:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574085.249739:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574085.249743:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249748:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574085.249752:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574085.249758:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574085.249766:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574085.249774:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574085.249785:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516653696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574085.249799:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574085.249804:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574085.249849:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516653696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574085.249860:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574085.249863:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574085.249869:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574085.249871:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574085.249874:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574085.249877:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574085.249881:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14772 00000400:00000200:0.0:1788574085.249887:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574085.249891:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574085.249895:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574085.249899:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249904:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574085.249908:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249918:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574085.249921:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574085.249923:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574085.249925:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574085.249926:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574085.249928:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574085.249932:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574085.249938:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249942:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574085.249946:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249949:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574085.249954:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574085.249960:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574085.249961:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574085.249963:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574085.249965:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574085.249966:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249969:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574085.249971:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574085.249973:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249976:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574085.249980:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574085.249985:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574085.249991:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574085.249992:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574085.249993:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574085.249994:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574085.249996:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574085.250001:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574085.250004:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574085.250010:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516653824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574085.250041:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574085.250045:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574085.250051:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516653824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574085.250066:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574085.250071:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14782) 00000100:00000001:0.0:1788574085.760919:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574085.760926:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574085.760930:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574085.760943:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574085.760947:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574085.760952:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574085.760965:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574085.760968:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574085.760972:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574086.465059:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574086.465062:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574086.465066:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574086.784943:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574086.784955:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574086.784963:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574086.784971:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574086.784977:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574086.784982:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574086.784987:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574086.784992:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574086.784999:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574087.808957:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574087.808964:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574087.808970:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574087.808993:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574087.809000:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574087.809007:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574087.809012:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574087.809034:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574087.809038:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574088.832945:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574088.832952:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574088.832956:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574088.832964:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574088.832971:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574088.832976:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574088.832993:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574088.832997:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574088.833001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574089.088904:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309448941, at 4309448960 00000100:00000010:0.0:1788574089.088922:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7720. 00000020:00000001:0.0:1788574089.537000:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574089.537005:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574089.537009:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574089.856909:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574089.856916:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574089.856921:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574089.856927:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574089.856933:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574089.856939:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574089.856954:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574089.856958:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574089.856962:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574090.880922:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574090.880930:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574090.880933:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574090.880938:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574090.880945:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574090.880949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574090.880955:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574090.880959:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574090.880963:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574090.880993:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574090.880997:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574090.880999:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574090.881011:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574090.881013:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574090.881015:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574090.881017:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574090.881018:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574090.881020:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574091.585004:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574091.585008:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574091.585011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574091.904903:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574091.904910:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574091.904914:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574091.904920:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574091.904925:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574091.904932:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574091.904950:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574091.904954:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574091.904957:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574092.928924:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574092.928933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574092.928937:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574092.928947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574092.928954:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574092.928959:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574092.928966:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574092.928970:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574092.928975:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574093.952898:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574093.952904:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574093.952908:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574093.952918:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574093.952927:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574093.952936:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574093.952944:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574093.952957:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574093.952965:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574093.953003:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574093.953008:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574093.953018:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574093.953022:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574093.953025:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574093.953028:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574093.953031:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574093.953033:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574093.953036:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574094.208882:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309454061, at 4309454080 00000100:00000010:0.0:1788574094.208898:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea74e0. 00000020:00000001:0.0:1788574094.657026:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574094.657030:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574094.657033:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574094.976998:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574094.977005:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574094.977009:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574094.977016:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574094.977023:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574094.977032:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574094.977059:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574094.977063:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574094.977068:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574095.488920:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574095.488931:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574095.488934:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574095.488939:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574095.488944:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574095.488954:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14782 00000400:00000200:0.0:1788574095.488966:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574095.488974:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574095.488978:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574095.488983:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.488987:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574095.488990:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.488993:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574095.489021:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574095.489023:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574095.489025:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574095.489028:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574095.489030:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574095.489032:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574095.489035:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489036:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574095.489037:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489039:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574095.489041:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574095.489044:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489047:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574095.489050:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574095.489053:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574095.489056:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.489058:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574095.489059:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574095.489068:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574095.489072:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574095.489076:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574095.489077:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.489080:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574095.489083:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574095.489088:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574095.489096:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574095.489104:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574095.489113:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516654592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574095.489123:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574095.489127:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574095.489131:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516654592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574095.489154:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574095.489157:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574095.489160:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574095.489161:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574095.489163:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574095.489165:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574095.489168:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14782 00000400:00000200:0.0:1788574095.489172:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574095.489175:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574095.489177:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574095.489179:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489181:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574095.489183:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489188:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574095.489191:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574095.489193:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574095.489197:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574095.489210:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574095.489215:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574095.489221:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574095.489229:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489235:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574095.489241:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489244:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574095.489250:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574095.489257:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489262:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574095.489272:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574095.489296:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574095.489300:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.489303:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574095.489309:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574095.489313:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.489318:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574095.489325:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574095.489331:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574095.489338:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574095.489342:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.489346:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574095.489348:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574095.489352:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574095.489360:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574095.489367:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574095.489373:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516654720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574095.489380:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574095.489382:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574095.489385:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516654720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574095.489393:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574095.489397:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574095.489405:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574095.489409:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574095.489421:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574095.489427:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574095.489434:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14782 00000400:00000200:0.0:1788574095.489443:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574095.489452:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574095.489457:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574095.489478:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489482:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574095.489487:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489491:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574095.489494:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574095.489497:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574095.489500:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574095.489505:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574095.489508:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574095.489513:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574095.489519:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489521:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574095.489526:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489531:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574095.489535:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574095.489537:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489540:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574095.489545:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574095.489549:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574095.489553:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.489555:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574095.489558:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574095.489562:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.489563:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574095.489567:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574095.489571:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574095.489575:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574095.489577:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.489580:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574095.489582:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574095.489586:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574095.489596:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574095.489605:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574095.489614:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516654848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574095.489630:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574095.489634:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574095.489643:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516654848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574095.489654:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574095.489657:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574095.489665:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574095.489668:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574095.489671:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574095.489676:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574095.489680:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14782 00000400:00000200:0.0:1788574095.489687:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574095.489693:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574095.489697:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574095.489703:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489707:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574095.489712:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489714:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574095.489717:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574095.489720:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574095.489722:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574095.489726:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574095.489730:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574095.489736:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574095.489739:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489740:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574095.489750:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489754:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574095.489756:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574095.489760:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574095.489763:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574095.489765:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574095.489776:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574095.489781:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.489788:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574095.489794:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574095.489801:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.489806:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574095.489860:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574095.489865:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574095.489871:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574095.489875:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.489880:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574095.489884:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574095.489887:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574095.489893:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574095.489899:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574095.489906:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516654976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574095.489915:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574095.489919:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574095.489925:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516654976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574095.489946:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574095.489954:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574095.489962:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574095.489966:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574095.489977:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574095.489984:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574095.489991:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14782 00000400:00000200:0.0:1788574095.490004:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574095.490014:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574095.490020:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574095.490027:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.490032:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574095.490039:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.490042:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574095.490047:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574095.490051:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574095.490055:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574095.490060:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574095.490064:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574095.490070:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574095.490079:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.490085:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574095.490090:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574095.490093:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574095.490098:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574095.490108:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574095.490113:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574095.490117:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574095.490122:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574095.490127:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.490132:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574095.490135:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574095.490141:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.490145:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574095.490157:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574095.490161:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574095.490166:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574095.490169:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574095.490173:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574095.490176:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574095.490180:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574095.490185:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574095.490191:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574095.490200:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516655104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574095.490211:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574095.490216:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574095.490228:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516655104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574095.490254:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574095.490261:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14793) 00000100:00000001:0.0:1788574096.000941:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574096.000950:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574096.000954:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574096.000962:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574096.000971:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574096.000980:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574096.001008:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574096.001012:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574096.001020:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574096.705054:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574096.705058:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574096.705061:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574097.024962:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574097.024970:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574097.024974:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574097.024981:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574097.024988:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574097.024991:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574097.025018:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574097.025020:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574097.025022:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574098.048940:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574098.048950:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574098.048954:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574098.048959:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574098.048969:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574098.048974:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574098.048980:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574098.048987:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574098.048994:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574098.049024:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574098.049035:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574098.049039:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574098.049049:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574098.049053:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574098.049060:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574098.049065:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574098.049069:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574098.049071:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574099.072983:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574099.072994:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574099.073008:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574099.073026:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574099.073041:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574099.073049:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574099.073056:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574099.073058:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574099.073061:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574099.328912:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309459181, at 4309459200 00000100:00000010:0.0:1788574099.328931:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea78a0. 00000020:00000001:0.0:1788574099.777057:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574099.777063:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574099.777066:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574100.096988:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574100.096999:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574100.097027:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574100.097032:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574100.097041:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574100.097044:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574100.097047:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574100.097049:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574100.097053:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574100.608946:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574100.608975:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574100.608979:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574100.608985:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574100.608992:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574100.609002:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14788 00000400:00000200:0.0:1788574100.609015:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574100.609024:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574100.609028:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574100.609033:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609038:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574100.609041:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609044:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574100.609049:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574100.609051:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574100.609054:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574100.609058:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574100.609060:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574100.609062:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574100.609065:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609067:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574100.609069:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609070:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574100.609080:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574100.609084:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609121:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574100.609124:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574100.609133:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574100.609136:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609139:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574100.609141:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574100.609148:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574100.609152:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574100.609157:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574100.609158:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609162:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574100.609165:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574100.609170:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574100.609183:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574100.609191:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574100.609201:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516655232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574100.609228:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574100.609232:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574100.609237:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516655232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574100.609279:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574100.609283:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574100.609287:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574100.609289:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574100.609291:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574100.609294:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574100.609298:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14788 00000400:00000200:0.0:1788574100.609303:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574100.609315:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574100.609318:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574100.609321:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609330:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574100.609332:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609335:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574100.609338:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574100.609340:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574100.609341:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574100.609344:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574100.609346:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574100.609349:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574100.609352:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609355:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574100.609357:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609360:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574100.609362:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574100.609365:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609367:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574100.609369:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574100.609377:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574100.609379:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609381:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574100.609386:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574100.609390:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609392:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574100.609395:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574100.609398:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574100.609400:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574100.609401:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609404:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574100.609405:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574100.609407:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574100.609411:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574100.609415:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574100.609427:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516655360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574100.609436:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574100.609439:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574100.609443:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516655360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574100.609460:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574100.609464:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574100.609467:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574100.609481:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574100.609483:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574100.609486:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574100.609489:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14788 00000400:00000200:0.0:1788574100.609493:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574100.609497:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574100.609499:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574100.609502:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609505:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574100.609507:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609509:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574100.609511:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574100.609513:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574100.609515:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574100.609517:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574100.609519:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574100.609521:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574100.609525:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609527:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574100.609529:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609536:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574100.609539:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574100.609542:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609545:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574100.609547:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574100.609550:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574100.609553:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609555:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574100.609558:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574100.609560:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609562:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574100.609566:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574100.609568:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574100.609571:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574100.609572:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609575:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574100.609576:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574100.609579:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574100.609583:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574100.609587:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574100.609592:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516655488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574100.609600:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574100.609602:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574100.609606:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516655488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574100.609615:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574100.609618:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574100.609622:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574100.609624:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574100.609625:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574100.609633:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574100.609652:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14788 00000400:00000200:0.0:1788574100.609657:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574100.609663:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574100.609666:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574100.609670:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609673:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574100.609675:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609678:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574100.609680:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574100.609682:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574100.609684:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574100.609686:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574100.609689:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574100.609692:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574100.609696:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609698:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574100.609701:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609703:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574100.609707:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574100.609710:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609714:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574100.609716:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574100.609719:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574100.609722:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609724:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574100.609744:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574100.609747:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609749:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574100.609754:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574100.609761:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574100.609764:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574100.609766:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609768:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574100.609770:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574100.609772:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574100.609776:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574100.609780:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574100.609785:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516655616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574100.609793:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574100.609796:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574100.609800:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516655616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574100.609810:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574100.609814:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574100.609818:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574100.609820:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574100.609822:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574100.609825:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574100.609828:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14788 00000400:00000200:0.0:1788574100.609833:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574100.609837:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574100.609874:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574100.609878:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609880:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574100.609883:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609886:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574100.609888:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574100.609891:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574100.609899:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574100.609902:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574100.609904:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574100.609907:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574100.609911:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609913:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574100.609916:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609919:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574100.609922:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574100.609925:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574100.609928:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574100.609931:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574100.609935:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574100.609938:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609941:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574100.609944:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574100.609948:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609950:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574100.609955:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574100.609958:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574100.609961:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574100.609962:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574100.609965:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574100.609966:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574100.609969:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574100.609973:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574100.609977:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574100.609982:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516655744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574100.609991:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574100.609993:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574100.609997:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516655744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574100.610010:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574100.610016:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14798) 00000020:00000001:0.0:1788574101.825018:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574101.825021:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574101.825024:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574102.145019:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574102.145026:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574102.145028:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574102.145032:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574102.145036:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574102.145037:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574102.145039:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574102.145041:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574102.145044:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574102.145082:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574102.145088:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574102.145093:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574102.145099:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574102.145105:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574102.145113:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574102.145131:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574102.145134:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574102.145136:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574103.169005:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574103.169010:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574103.169013:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574103.169017:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574103.169025:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574103.169031:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574103.169039:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574103.169042:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574103.169048:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574103.169134:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574103.169141:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574103.169143:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574103.169147:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574103.169166:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574103.169170:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574103.169172:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574103.169174:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574103.169177:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574104.192908:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574104.192914:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574104.192919:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574104.192926:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574104.192934:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574104.192941:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574104.192954:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574104.192962:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574104.192967:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574104.192986:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574104.192990:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574104.192992:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574104.192994:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574104.192997:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574104.192999:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574104.193009:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574104.193013:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574104.193016:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574104.448884:0:265679:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309464301, at 4309464320 00000100:00000010:0.0:1788574104.448897:0:265679:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea71e0. 00000020:00000001:0.0:1788574104.896986:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574104.896990:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574104.896992:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574105.216936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574105.216941:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574105.216945:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574105.216952:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574105.216959:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574105.216965:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574105.216984:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574105.216987:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574105.216991:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574106.240914:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574106.240919:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574106.240923:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574106.240929:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574106.240935:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574106.240941:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574106.240954:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574106.240957:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574106.240960:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574106.945070:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574106.945073:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574106.945076:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574107.264887:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574107.264898:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574107.264902:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574107.264907:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574107.264913:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574107.264918:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574107.264923:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574107.264929:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574107.264938:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574107.264990:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574107.264994:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574107.264996:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574107.265003:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574107.265017:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574107.265021:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574107.265031:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574107.265035:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574107.265039:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574109.312942:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574109.312951:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574109.312954:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574109.312969:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574109.312983:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574109.312988:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574109.312997:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574109.313001:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574109.313022:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574109.313077:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574109.313081:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574109.313087:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574109.313091:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574109.313095:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574109.313098:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574109.313101:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574109.313103:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574109.313106:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574109.568922:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309469422, at 4309469440 00000100:00000010:0.0:1788574109.568966:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7840. 00000020:00000001:0.0:1788574110.017030:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574110.017034:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574110.017037:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574110.336906:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574110.336911:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574110.336915:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574110.336919:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574110.336924:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574110.336929:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574110.336956:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574110.336960:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574110.336964:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574110.848903:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574110.848913:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574110.848916:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574110.848919:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574110.848926:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574110.848940:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14798 00000400:00000200:0.0:1788574110.848968:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574110.848974:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574110.848988:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574110.848992:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849007:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574110.849009:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849012:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574110.849015:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574110.849016:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574110.849018:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574110.849021:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574110.849022:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574110.849024:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574110.849026:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849027:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574110.849029:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849030:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574110.849032:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574110.849035:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849037:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574110.849039:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574110.849041:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574110.849044:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849045:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574110.849047:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574110.849052:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574110.849055:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574110.849058:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574110.849059:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849062:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574110.849064:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574110.849068:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574110.849074:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574110.849080:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574110.849089:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516656512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574110.849115:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574110.849121:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574110.849133:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516656512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574110.849157:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574110.849160:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574110.849164:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574110.849165:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574110.849168:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574110.849171:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574110.849174:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14798 00000400:00000200:0.0:1788574110.849179:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574110.849186:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574110.849191:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574110.849198:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849204:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574110.849209:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849213:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574110.849220:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574110.849222:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574110.849226:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574110.849231:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574110.849237:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574110.849243:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574110.849252:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849258:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574110.849263:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849269:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574110.849281:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574110.849287:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849290:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574110.849292:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574110.849297:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574110.849299:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849304:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574110.849308:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574110.849313:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849316:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574110.849323:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574110.849327:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574110.849331:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574110.849334:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849337:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574110.849339:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574110.849342:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574110.849348:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574110.849352:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574110.849359:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516656640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574110.849372:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574110.849375:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574110.849383:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516656640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574110.849396:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574110.849400:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574110.849407:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574110.849410:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574110.849413:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574110.849418:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574110.849429:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14798 00000400:00000200:0.0:1788574110.849437:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574110.849442:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574110.849445:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574110.849451:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849454:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574110.849459:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849463:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574110.849466:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574110.849469:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574110.849472:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574110.849476:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574110.849479:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574110.849482:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574110.849487:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849490:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574110.849494:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849499:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574110.849501:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574110.849504:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849507:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574110.849510:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574110.849513:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574110.849517:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849519:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574110.849523:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574110.849528:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849531:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574110.849537:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574110.849539:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574110.849541:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574110.849548:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849550:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574110.849553:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574110.849556:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574110.849562:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574110.849567:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574110.849573:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516656768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574110.849580:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574110.849583:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574110.849588:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516656768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574110.849598:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574110.849600:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574110.849602:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574110.849603:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574110.849604:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574110.849606:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574110.849610:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14798 00000400:00000200:0.0:1788574110.849615:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574110.849620:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574110.849622:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574110.849628:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849635:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574110.849641:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849647:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574110.849653:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574110.849656:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574110.849660:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574110.849665:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574110.849671:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574110.849684:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574110.849691:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849694:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574110.849706:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849708:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574110.849711:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574110.849715:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849718:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574110.849721:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574110.849725:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574110.849729:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849733:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574110.849736:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574110.849742:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849747:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574110.849752:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574110.849756:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574110.849761:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574110.849764:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849769:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574110.849771:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574110.849774:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574110.849779:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574110.849785:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574110.849792:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516656896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574110.849806:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574110.849825:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574110.849829:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516656896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574110.849844:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574110.849854:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574110.849858:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574110.849861:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574110.849864:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574110.849868:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574110.849873:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14798 00000400:00000200:0.0:1788574110.849879:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574110.849885:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574110.849890:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574110.849893:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849904:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574110.849906:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849907:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574110.849908:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574110.849912:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574110.849913:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574110.849918:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574110.849922:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574110.849926:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574110.849932:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849936:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574110.849941:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849944:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574110.849947:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574110.849952:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574110.849955:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574110.849958:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574110.849960:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574110.849961:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849963:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574110.849971:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574110.849974:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.849990:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574110.849995:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574110.849998:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574110.850002:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574110.850004:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574110.850007:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574110.850010:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574110.850013:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574110.850028:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574110.850033:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574110.850040:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516657024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574110.850054:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574110.850058:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574110.850067:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516657024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574110.850081:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574110.850088:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14808) 00000100:00000001:0.0:1788574111.360948:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574111.360953:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574111.360957:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574111.360974:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574111.360979:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574111.360984:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574111.360989:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574111.360991:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574111.360994:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574112.064956:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574112.064960:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574112.064962:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574112.384922:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574112.384945:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574112.384965:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574112.384977:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574112.384985:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574112.384989:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574112.385006:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574112.385010:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574112.385013:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574112.385032:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574112.385035:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574112.385036:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574112.385038:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574112.385040:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574112.385041:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574112.385043:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574112.385045:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574112.385047:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574113.408924:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574113.408928:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574113.408932:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574113.408939:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574113.408944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574113.408948:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574113.408952:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574113.408956:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574113.408962:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574113.409013:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574113.409017:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574113.409018:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574113.409020:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574113.409022:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574113.409024:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574113.409026:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574113.409027:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574113.409030:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574114.688941:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309474541, at 4309474560 00000100:00000010:0.0:1788574114.688965:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7d80. 00000020:00000001:0.0:1788574115.137039:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574115.137062:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574115.137066:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574115.456980:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574115.456986:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574115.456990:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574115.456995:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574115.457000:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574115.457005:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574115.457010:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574115.457014:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574115.457019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574116.480934:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574116.480940:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574116.480944:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574116.480949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574116.480955:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574116.480963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574116.480980:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574116.480986:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574116.480993:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574117.185032:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574117.185036:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574117.185039:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574117.504891:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574117.504897:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574117.504901:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574117.504908:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574117.504915:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574117.504920:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574117.504932:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574117.504936:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574117.504941:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574117.504976:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574117.504980:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574117.504982:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574117.504984:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574117.504987:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574117.504989:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574117.505006:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574117.505007:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574117.505009:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574118.528918:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574118.528926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574118.528929:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574118.528935:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574118.528941:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574118.528947:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574118.528954:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574118.528958:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574118.528962:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574118.528987:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574118.528989:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574118.528990:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574118.528992:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574118.528994:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574118.528997:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574118.529007:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574118.529010:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574118.529013:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574119.808910:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309479661, at 4309479680 00000100:00000010:0.0:1788574119.808927:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f00. 00000020:00000001:0.0:1788574120.256954:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574120.256958:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574120.256960:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574120.576926:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574120.576933:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574120.576937:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574120.576944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574120.576952:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574120.576956:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574120.576973:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574120.576976:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574120.576980:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574121.088919:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574121.088928:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574121.088946:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574121.088949:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574121.088953:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574121.088959:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14808 00000400:00000200:0.0:1788574121.088966:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574121.088971:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574121.088974:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574121.088983:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.088986:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574121.088987:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.088989:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574121.088992:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574121.088994:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574121.088996:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574121.088999:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574121.089000:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574121.089002:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574121.089004:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089006:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574121.089007:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089009:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574121.089011:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574121.089013:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089015:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574121.089016:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574121.089019:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574121.089021:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089022:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574121.089024:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574121.089030:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574121.089033:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574121.089042:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574121.089043:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089045:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574121.089048:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574121.089052:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574121.089058:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574121.089063:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574121.089070:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516657792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574121.089077:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574121.089081:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574121.089084:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516657792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574121.089100:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574121.089102:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574121.089105:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574121.089106:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574121.089107:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574121.089109:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574121.089111:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14808 00000400:00000200:0.0:1788574121.089114:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574121.089117:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574121.089119:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574121.089122:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089123:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574121.089128:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089131:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574121.089134:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574121.089136:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574121.089139:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574121.089141:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574121.089150:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574121.089156:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574121.089161:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089163:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574121.089164:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089165:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574121.089167:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574121.089172:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089175:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574121.089177:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574121.089184:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574121.089188:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089191:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574121.089197:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574121.089202:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089204:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574121.089209:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574121.089213:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574121.089214:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574121.089215:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089217:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574121.089218:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574121.089222:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574121.089227:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574121.089231:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574121.089238:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516657920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574121.089244:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574121.089246:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574121.089249:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516657920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574121.089271:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574121.089277:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574121.089289:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574121.089293:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574121.089296:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574121.089300:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574121.089305:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14808 00000400:00000200:0.0:1788574121.089313:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574121.089319:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574121.089323:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574121.089328:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089333:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574121.089335:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089337:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574121.089342:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574121.089345:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574121.089346:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574121.089348:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574121.089351:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574121.089356:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574121.089361:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089365:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574121.089370:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089373:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574121.089377:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574121.089380:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089381:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574121.089389:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574121.089393:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574121.089399:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089404:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574121.089407:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574121.089409:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089413:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574121.089418:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574121.089428:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574121.089430:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574121.089434:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089435:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574121.089436:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574121.089440:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574121.089452:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574121.089462:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574121.089471:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516658048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574121.089486:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574121.089489:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574121.089495:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516658048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574121.089501:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574121.089505:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574121.089510:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574121.089513:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574121.089515:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574121.089520:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574121.089525:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14808 00000400:00000200:0.0:1788574121.089530:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574121.089537:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574121.089542:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574121.089552:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089558:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574121.089564:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089567:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574121.089571:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574121.089572:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574121.089574:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574121.089577:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574121.089579:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574121.089581:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574121.089584:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089587:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574121.089589:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089593:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574121.089596:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574121.089600:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089604:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574121.089608:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574121.089613:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574121.089616:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089625:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574121.089629:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574121.089633:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089636:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574121.089641:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574121.089645:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574121.089649:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574121.089651:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089656:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574121.089658:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574121.089661:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574121.089678:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574121.089688:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574121.089694:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516658176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574121.089708:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574121.089711:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574121.089717:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516658176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574121.089741:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574121.089743:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574121.089745:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574121.089746:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574121.089747:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574121.089750:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574121.089755:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14808 00000400:00000200:0.0:1788574121.089760:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574121.089766:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574121.089771:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574121.089783:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089791:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574121.089796:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089800:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574121.089806:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574121.089837:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574121.089843:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574121.089849:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574121.089856:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574121.089859:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574121.089865:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089870:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574121.089878:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089882:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574121.089886:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574121.089891:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574121.089895:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574121.089897:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574121.089902:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574121.089908:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089911:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574121.089915:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574121.089921:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089922:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574121.089925:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574121.089928:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574121.089932:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574121.089935:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574121.089939:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574121.089942:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574121.089945:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574121.089951:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574121.089956:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574121.089966:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516658304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574121.089982:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574121.089984:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574121.089991:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516658304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574121.090007:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574121.090014:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14818) 00000100:00000001:0.0:1788574121.600892:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574121.600898:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574121.600902:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574121.600918:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574121.600924:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574121.600928:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574121.600932:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574121.600935:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574121.600938:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574122.304952:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574122.304959:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574122.304962:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574122.624924:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574122.624931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574122.624934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574122.624942:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574122.624949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574122.624953:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574122.624958:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574122.624961:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574122.624966:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574123.648908:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574123.648918:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574123.648923:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574123.648931:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574123.648939:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574123.648946:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574123.648961:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574123.648964:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574123.648968:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574124.672935:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574124.672942:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574124.672946:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574124.672953:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574124.672959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574124.672964:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574124.672968:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574124.672971:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574124.672974:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574124.928892:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309484781, at 4309484800 00000100:00000010:0.0:1788574124.928907:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ea0. 00000020:00000001:0.0:1788574125.376996:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574125.376999:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574125.377002:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574125.696938:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574125.696943:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574125.696948:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574125.696952:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574125.696957:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574125.696962:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574125.696983:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574125.696986:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574125.696989:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574126.720892:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574126.720897:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574126.720902:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574126.720908:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574126.720914:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574126.720920:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574126.720933:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574126.720937:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574126.720939:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574127.425009:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574127.425013:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574127.425016:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574127.745093:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574127.745098:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574127.745101:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574127.745107:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574127.745111:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574127.745114:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574127.745116:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574127.745117:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574127.745119:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574128.768903:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574128.768909:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574128.768913:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574128.768919:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574128.768927:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574128.768950:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574128.768955:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574128.768960:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574128.768966:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574128.769044:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574128.769047:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574128.769048:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574128.769051:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574128.769053:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574128.769054:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574128.769056:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574128.769057:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574128.769058:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574130.049029:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309489901, at 4309489920 00000100:00000010:0.0:1788574130.049035:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab600. 00000020:00000001:0.0:1788574130.497024:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574130.497029:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574130.497032:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574130.816904:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574130.816910:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574130.816915:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574130.816921:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574130.816925:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574130.816931:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574130.816945:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574130.816948:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574130.816951:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574131.840926:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574131.840931:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574131.840934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574131.840941:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574131.840945:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574131.840950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574131.840955:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574131.840961:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574131.840967:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574132.545030:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574132.545053:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574132.545056:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574132.864944:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574132.864949:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574132.864952:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574132.864956:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574132.864962:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574132.864966:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574132.864972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574132.864975:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574132.864980:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574133.888935:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574133.888944:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574133.888948:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574133.888956:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574133.888964:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574133.888970:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574133.888978:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574133.888981:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574133.888984:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574133.889016:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574133.889020:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574133.889024:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574133.889029:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574133.889032:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574133.889034:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574133.889035:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574133.889037:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574133.889039:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574134.912916:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574134.912922:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574134.912925:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574134.912930:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574134.912935:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574134.912939:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574134.912960:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574134.912965:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574134.912970:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574135.168934:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309495021, at 4309495040 00000100:00000010:0.0:1788574135.168951:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab1e0. 00000020:00000001:0.0:1788574135.617041:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574135.617045:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574135.617048:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574135.937031:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574135.937037:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574135.937040:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574135.937044:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574135.937048:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574135.937052:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574135.937057:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574135.937059:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574135.937064:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574136.960921:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574136.960927:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574136.960929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574136.960935:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574136.960942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574136.960946:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574136.960949:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574136.960953:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574136.960962:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574137.665033:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574137.665036:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574137.665039:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574137.984913:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574137.984923:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574137.984929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574137.984936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574137.984943:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574137.984951:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574137.984961:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574137.984965:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574137.984972:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574139.009169:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574139.009173:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574139.009198:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574139.009200:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574139.009202:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574139.009203:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574139.009205:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574139.009206:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574139.009207:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574140.032896:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574140.032902:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574140.032906:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574140.032913:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574140.032921:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574140.032926:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574140.032937:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574140.032940:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574140.032945:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574140.032973:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574140.032977:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574140.032978:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574140.032980:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574140.032983:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574140.032985:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574140.032987:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574140.032988:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574140.032990:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574140.288898:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309500142, at 4309500160 00000100:00000010:0.0:1788574140.288916:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab300. 00000020:00000001:0.0:1788574140.736982:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574140.736986:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574140.736990:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574141.056987:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574141.056991:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574141.056993:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574141.056995:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574141.056998:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574141.057000:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574141.057002:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574141.057013:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574141.057017:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574142.080927:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574142.080936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574142.080941:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574142.080947:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574142.080954:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574142.080959:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574142.080978:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574142.080981:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574142.080984:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574142.785054:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574142.785058:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574142.785060:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574143.104941:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574143.104949:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574143.104952:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574143.104962:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574143.104969:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574143.104973:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574143.104978:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574143.104983:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574143.104987:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574144.128947:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574144.128953:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574144.128957:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574144.128978:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574144.128984:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574144.128988:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574144.129017:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574144.129020:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574144.129023:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574145.152925:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574145.152943:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574145.152947:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574145.152953:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574145.152959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574145.152964:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574145.152984:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574145.152987:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574145.152990:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574145.408886:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309505261, at 4309505280 00000100:00000010:0.0:1788574145.408905:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab7e0. 00000020:00000001:0.0:1788574145.856963:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574145.856967:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574145.856970:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574146.176949:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574146.176957:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574146.176962:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574146.176968:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574146.176976:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574146.176981:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574146.176988:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574146.176994:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574146.177002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574147.200930:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574147.200936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574147.200941:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574147.200946:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574147.200953:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574147.200960:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574147.200973:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574147.200978:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574147.200980:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574147.905034:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574147.905039:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574147.905042:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574148.224985:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574148.224991:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574148.224995:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574148.225000:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574148.225004:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574148.225007:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574148.225555:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574148.225561:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574148.225576:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574148.737116:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788574149.248930:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574149.248936:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574149.248940:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574149.248948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574149.248953:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574149.248969:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574149.248976:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574149.248980:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574149.248987:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574150.272927:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574150.272932:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574150.272938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574150.272944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574150.272950:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574150.272956:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574150.272961:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574150.272964:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574150.272967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574150.528886:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309510381, at 4309510400 00000100:00000010:0.0:1788574150.528901:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab180. 00000020:00000001:0.0:1788574150.976942:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574150.976946:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574150.976949:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574151.296974:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574151.296980:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574151.296985:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574151.296994:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574151.297003:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574151.297010:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574151.297017:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574151.297025:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574151.297030:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574151.297056:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574151.297062:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574151.297064:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574151.297084:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574151.297088:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574151.297091:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574151.297095:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574151.297097:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574151.297100:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574152.320976:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574152.320984:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574152.320988:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574152.321001:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574152.321016:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574152.321027:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574152.321041:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574152.321049:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574152.321055:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574153.025042:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574153.025049:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574153.025055:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574153.344957:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574153.344966:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574153.344972:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574153.344984:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574153.344995:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574153.345000:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574153.345012:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574153.345016:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574153.345021:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574154.368954:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574154.368964:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574154.368968:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574154.368975:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574154.368980:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574154.368986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574154.368998:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574154.369006:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574154.369016:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574155.392942:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574155.392950:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574155.392975:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574155.392982:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574155.392986:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574155.392990:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574155.393002:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574155.393006:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574155.393009:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574155.648961:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309515502, at 4309515520 00000100:00000010:0.0:1788574155.648989:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab3c0. 00000020:00000001:0.0:1788574156.097042:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574156.097047:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574156.097051:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574156.416923:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574156.416939:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574156.416943:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574156.416954:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574156.416961:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574156.416968:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574156.416973:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574156.416978:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574156.416981:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574157.440922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574157.440929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574157.440934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574157.440938:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574157.440946:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574157.440955:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574157.440978:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574157.440983:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574157.440989:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574158.144970:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574158.144974:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574158.144976:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574158.464880:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574158.464913:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574158.464915:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574158.464918:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574158.464937:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574158.464939:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574158.464941:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574158.464943:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574158.464952:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574158.464990:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574158.464993:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574158.464995:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574158.464997:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574158.464999:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574158.465001:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574158.465004:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574158.465005:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574158.465008:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574159.488948:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574159.488956:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574159.488961:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574159.488969:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574159.488973:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574159.488977:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574159.488993:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574159.488998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574159.489007:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574160.512893:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574160.512899:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574160.512902:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574160.512908:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574160.512915:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574160.512919:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574160.512925:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574160.512928:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574160.512933:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574160.512950:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574160.512954:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574160.512956:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574160.512958:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574160.512961:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574160.512963:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574160.512972:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574160.512973:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574160.512975:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574160.768881:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309520622, at 4309520640 00000100:00000010:0.0:1788574160.768895:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abba0. 00000020:00000001:0.0:1788574161.216978:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574161.216982:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574161.216985:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574162.560963:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574162.560971:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574162.560975:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574162.560982:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574162.560989:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574162.560994:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574162.561014:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574162.561019:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574162.561025:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574163.264998:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574163.265003:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574163.265006:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574163.584928:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574163.584935:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574163.584940:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574163.584946:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574163.584952:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574163.584958:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574163.584962:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574163.584964:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574163.584967:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574164.608931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574164.608938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574164.608942:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574164.608949:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574164.608955:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574164.608961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574164.608974:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574164.608977:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574164.608997:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574165.888907:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309525741, at 4309525760 00000100:00000010:0.0:1788574165.888920:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abc00. 00000020:00000001:0.0:1788574166.336951:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574166.336955:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574166.336958:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574166.656930:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574166.656934:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574166.656939:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574166.656944:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574166.656962:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574166.656967:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574166.656977:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574166.656982:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574166.656984:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574167.168920:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574167.168932:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574167.168936:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574167.168939:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574167.168945:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574167.168952:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14854 00000400:00000200:0.0:1788574167.168963:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574167.168969:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574167.168973:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574167.168977:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.168981:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574167.168983:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.168986:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574167.168990:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574167.168993:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574167.168996:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574167.168999:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574167.169002:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574167.169022:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574167.169026:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169029:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574167.169031:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169033:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574167.169036:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574167.169040:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169043:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574167.169046:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574167.169050:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574167.169053:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169055:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574167.169057:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574167.169063:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574167.169067:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574167.169071:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574167.169072:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169074:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574167.169077:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574167.169081:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574167.169086:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574167.169092:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574167.169099:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516663552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574167.169107:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574167.169110:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574167.169113:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516663552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574167.169132:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574167.169135:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574167.169137:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574167.169143:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574167.169144:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574167.169147:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574167.169149:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14854 00000400:00000200:0.0:1788574167.169152:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574167.169154:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574167.169157:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574167.169159:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169161:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574167.169164:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169165:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574167.169167:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574167.169168:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574167.169169:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574167.169171:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574167.169173:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574167.169177:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574167.169179:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169183:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574167.169186:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169189:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574167.169192:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574167.169198:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169202:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574167.169205:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574167.169211:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574167.169215:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169217:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574167.169221:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574167.169225:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169234:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574167.169238:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574167.169241:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574167.169243:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574167.169244:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169245:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574167.169247:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574167.169250:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574167.169258:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574167.169279:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574167.169284:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516663680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574167.169295:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574167.169298:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574167.169304:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516663680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574167.169320:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574167.169325:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574167.169334:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574167.169338:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574167.169342:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574167.169350:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574167.169356:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14854 00000400:00000200:0.0:1788574167.169366:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574167.169373:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574167.169377:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574167.169380:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169382:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574167.169386:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169389:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574167.169391:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574167.169401:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574167.169404:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574167.169409:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574167.169412:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574167.169415:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574167.169420:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169425:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574167.169430:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169433:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574167.169437:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574167.169440:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169444:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574167.169448:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574167.169453:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574167.169456:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169461:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574167.169464:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574167.169467:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169468:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574167.169474:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574167.169478:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574167.169484:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574167.169486:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169489:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574167.169492:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574167.169494:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574167.169497:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574167.169500:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574167.169505:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516663808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574167.169519:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574167.169522:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574167.169534:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516663808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574167.169544:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574167.169547:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574167.169553:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574167.169554:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574167.169556:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574167.169558:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574167.169563:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14854 00000400:00000200:0.0:1788574167.169567:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574167.169572:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574167.169575:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574167.169580:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169582:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574167.169584:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169588:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574167.169591:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574167.169593:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574167.169596:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574167.169599:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574167.169602:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574167.169606:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574167.169613:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169614:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574167.169617:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169621:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574167.169626:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574167.169631:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169634:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574167.169640:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574167.169644:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574167.169649:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169651:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574167.169654:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574167.169659:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169661:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574167.169665:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574167.169670:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574167.169673:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574167.169676:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169679:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574167.169681:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574167.169684:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574167.169688:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574167.169692:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574167.169698:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516663936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574167.169708:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574167.169710:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574167.169714:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516663936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574167.169728:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574167.169732:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574167.169735:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574167.169739:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574167.169741:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574167.169747:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574167.169752:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14854 00000400:00000200:0.0:1788574167.169760:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574167.169769:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574167.169775:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574167.169789:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169793:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574167.169797:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169801:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574167.169803:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574167.169805:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574167.169860:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574167.169864:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574167.169867:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574167.169869:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574167.169873:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169876:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574167.169880:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169883:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574167.169885:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574167.169888:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574167.169892:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574167.169897:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574167.169901:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574167.169906:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169908:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574167.169911:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574167.169918:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169922:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574167.169929:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574167.169935:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574167.169942:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574167.169946:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574167.169951:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574167.169954:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574167.169958:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574167.169965:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574167.169978:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574167.169987:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516664064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574167.169994:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574167.169999:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574167.170007:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516664064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574167.170022:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574167.170029:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14864) 00000100:00000001:0.0:1788574167.680896:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574167.680903:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574167.680908:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574167.680914:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574167.680919:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574167.680924:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574167.680940:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574167.680943:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574167.680946:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574168.385039:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574168.385043:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574168.385047:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574168.704962:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574168.704970:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574168.704974:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574168.704980:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574168.704986:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574168.704992:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574168.704996:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574168.705001:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574168.705004:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574168.705041:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574168.705046:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574168.705049:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574168.705054:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574168.705058:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574168.705084:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574168.705088:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574168.705091:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574168.705096:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574169.728903:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574169.728909:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574169.728913:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574169.728921:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574169.728929:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574169.728935:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574169.728946:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574169.728952:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574169.728958:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574170.752947:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574170.752952:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574170.752956:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574170.752962:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574170.752966:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574170.752969:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574170.752977:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574170.752980:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574170.752983:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574171.008911:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309530861, at 4309530880 00000100:00000010:0.0:1788574171.008928:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab000. 00000020:00000001:0.0:1788574171.457069:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574171.457072:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574171.457075:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574171.776896:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574171.776903:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574171.776906:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574171.776914:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574171.776921:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574171.776926:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574171.776942:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574171.776947:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574171.776951:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574171.776972:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574171.776982:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574171.776984:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574171.776987:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574171.776990:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574171.776992:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574171.776994:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574171.776995:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574171.776998:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574172.800901:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574172.800909:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574172.800914:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574172.800921:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574172.800928:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574172.800934:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574172.800970:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574172.800975:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574172.800980:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574173.505012:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574173.505015:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574173.505017:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574173.824919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574173.824925:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574173.824930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574173.824939:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574173.824947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574173.824952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574173.824960:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574173.824964:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574173.824968:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574173.825183:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574173.825186:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574173.825188:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574173.825191:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574173.825194:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574173.825196:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574173.825199:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574173.825200:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574173.825203:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574174.848979:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574174.848991:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574174.848994:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574174.849012:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574174.849020:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574174.849026:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574174.849031:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574174.849035:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574174.849040:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574175.872891:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574175.872897:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574175.872900:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574175.872902:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574175.872905:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574175.872909:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574175.872921:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574175.872925:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574175.872927:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574176.128897:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309535982, at 4309536000 00000100:00000010:0.0:1788574176.128914:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab4e0. 00000020:00000001:0.0:1788574176.576989:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574176.576993:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574176.576995:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574176.896957:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574176.896961:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574176.896966:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574176.896974:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574176.896983:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574176.896988:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574176.896995:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574176.896998:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574176.897001:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574177.920887:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574177.920896:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574177.920900:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574177.920908:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574177.920915:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574177.920939:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574177.920950:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574177.920953:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574177.920956:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574178.625074:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574178.625087:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574178.625090:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574178.944962:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574178.944967:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574178.944970:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574178.944974:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574178.944977:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574178.944980:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574178.944983:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574178.944985:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574178.944988:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574179.968940:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574179.968950:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574179.968952:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574179.968958:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574179.968962:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574179.968966:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574179.968969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574179.968971:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574179.968976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574179.969011:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574179.969018:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574179.969023:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574179.969028:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574179.969034:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574179.969046:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574179.969065:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574179.969073:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574179.969076:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574180.992943:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574180.992951:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574180.992955:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574180.992959:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574180.992982:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574180.992989:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574180.992994:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574180.992999:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574180.993003:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574180.993219:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574180.993221:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574180.993222:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574180.993224:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574180.993225:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574180.993226:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574180.993228:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574180.993229:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574180.993230:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574181.248901:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309541101, at 4309541120 00000100:00000010:0.0:1788574181.248920:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7360. 00000020:00000001:0.0:1788574181.697103:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574181.697109:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574181.697113:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574182.016919:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574182.016926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574182.016931:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574182.016938:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574182.016945:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574182.016951:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574182.016981:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574182.016985:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574182.016988:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574183.744987:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574183.744992:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574183.744995:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574184.064940:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574184.064946:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574184.064951:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574184.064960:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574184.064976:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574184.064982:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574184.065003:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574184.065006:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574184.065011:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574185.088937:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574185.088942:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574185.088945:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574185.088951:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574185.088958:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574185.088966:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574185.088974:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574185.088977:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574185.088981:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574186.112898:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574186.112904:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574186.112907:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574186.112915:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574186.112919:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574186.112924:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574186.112928:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574186.112932:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574186.112938:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574186.113022:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574186.113025:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574186.113026:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574186.113029:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574186.113033:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574186.113035:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574186.113039:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574186.113042:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574186.113045:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574186.368875:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309546221, at 4309546240 00000100:00000010:0.0:1788574186.368889:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7600. 00000020:00000001:0.0:1788574186.817178:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574186.817182:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574186.817187:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574187.136927:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574187.136933:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574187.136950:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574187.136955:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574187.136960:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574187.136965:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574187.136972:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574187.136974:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574187.136977:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574187.649127:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788574188.160939:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574188.160946:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574188.160950:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574188.160956:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574188.160960:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574188.160963:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574188.160980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574188.160984:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574188.160987:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574188.865044:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574188.865047:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574188.865050:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574189.184930:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574189.184946:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574189.184950:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574189.184960:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574189.184966:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574189.184971:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574189.184977:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574189.184981:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574189.184986:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574189.185043:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574189.185047:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574189.185049:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574189.185052:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574189.185054:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574189.185057:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574189.185060:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574189.185062:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574189.185065:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574190.208902:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574190.208910:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574190.208915:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574190.208925:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574190.208935:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574190.208956:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574190.208976:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574190.208979:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574190.208981:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574191.232907:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574191.232916:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574191.232920:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574191.232928:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574191.232936:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574191.232941:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574191.232978:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574191.232980:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574191.232983:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574191.488894:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309551341, at 4309551360 00000100:00000010:0.0:1788574191.488907:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ba0. 00000020:00000001:0.0:1788574191.936944:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574191.936948:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574191.936951:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574192.256946:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574192.256951:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574192.256956:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574192.256964:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574192.256971:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574192.256977:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574192.256983:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574192.256988:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574192.256992:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574193.280923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574193.280931:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574193.280933:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574193.280939:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574193.280944:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574193.280967:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574193.280974:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574193.280979:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574193.280986:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574193.281029:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574193.281032:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574193.281034:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574193.281037:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574193.281040:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574193.281042:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574193.281044:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574193.281046:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574193.281048:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574193.985004:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574193.985009:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574193.985012:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574194.304903:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574194.304912:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574194.304916:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574194.304923:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574194.304929:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574194.304932:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574194.304942:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574194.304945:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574194.304951:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574194.304986:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574194.304989:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574194.304990:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574194.304992:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574194.304994:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574194.304996:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574194.304998:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574194.304999:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574194.305001:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574195.328875:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574195.328882:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574195.328886:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574195.328892:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574195.328911:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574195.328915:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574195.328918:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574195.328922:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574195.328926:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574196.353096:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574196.353100:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574196.353102:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574196.353105:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574196.353108:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574196.353111:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574196.353113:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574196.353115:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574196.353118:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574196.608950:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309556461, at 4309556480 00000100:00000010:0.0:1788574196.608966:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7420. 00000020:00000001:0.0:1788574197.057128:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574197.057132:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574197.057134:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574197.376931:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574197.376946:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574197.376958:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574197.376978:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574197.376996:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574197.377014:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574197.377034:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574197.377043:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574197.377048:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574198.400940:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574198.400963:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574198.400969:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574198.400988:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574198.401004:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574198.401019:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574198.401048:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574198.401053:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574198.401060:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574199.105066:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574199.105069:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574199.105072:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574199.425005:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574199.425011:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574199.425016:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574199.425023:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574199.425027:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574199.425035:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574199.425052:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574199.425060:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574199.425065:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574200.448902:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574200.448916:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574200.448920:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574200.448928:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574200.448936:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574200.448944:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574200.448952:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574200.448956:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574200.448961:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574201.473081:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574201.473088:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574201.473098:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574201.473112:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574201.473128:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574201.473134:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574201.473138:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574201.473141:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574201.473148:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574201.728893:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309561582, at 4309561600 00000100:00000010:0.0:1788574201.728935:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab0c0. 00000020:00000001:0.0:1788574202.177010:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574202.177018:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574202.177035:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574202.496942:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574202.496957:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574202.496985:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574202.496993:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574202.496999:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574202.497005:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574202.497022:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574202.497026:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574202.497029:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574203.521070:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574203.521076:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574203.521081:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574203.521089:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574203.521095:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574203.521102:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574203.521118:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574203.521127:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574203.521137:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574204.225005:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574204.225009:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574204.225012:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574204.544967:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574204.544976:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574204.544982:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574204.545000:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574204.545012:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574204.545025:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574204.545061:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574204.545066:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574204.545070:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574205.568939:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574205.568947:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574205.568951:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574205.568958:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574205.568965:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574205.568970:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574205.568974:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574205.568976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574205.568984:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574206.592912:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574206.592943:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574206.592946:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574206.592949:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574206.592952:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574206.592954:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574206.592956:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574206.592959:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574206.592965:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574206.848899:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309566702, at 4309566720 00000100:00000010:0.0:1788574206.848916:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7a80. 00000020:00000001:0.0:1788574207.297060:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574207.297067:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574207.297072:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574207.616978:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574207.616989:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574207.616993:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574207.617003:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574207.617018:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574207.617032:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574207.617074:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574207.617077:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574207.617082:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574208.640983:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574208.640995:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574208.641000:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574208.641008:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574208.641015:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574208.641022:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574208.641031:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574208.641039:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574208.641045:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574209.345005:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574209.345009:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574209.345011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574209.664920:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574209.664929:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574209.664934:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574209.664961:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574209.664968:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574209.664974:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574209.664989:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574209.664992:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574209.664996:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574210.688933:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574210.688938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574210.688942:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574210.688947:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574210.688953:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574210.688958:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574210.688972:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574210.688976:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574210.688980:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574211.712940:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574211.712945:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574211.712950:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574211.712955:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574211.712959:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574211.712963:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574211.713001:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574211.713004:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574211.713010:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574211.968912:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309571821, at 4309571840 00000100:00000010:0.0:1788574211.968927:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea79c0. 00000020:00000001:0.0:1788574212.416993:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574212.416996:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574212.416999:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574212.736903:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574212.736912:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574212.736916:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574212.736924:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574212.736930:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574212.736935:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574212.737052:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574212.737056:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574212.737078:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574213.760897:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574213.760905:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574213.760908:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574213.760914:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574213.760919:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574213.760925:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574213.760931:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574213.760936:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574213.760941:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574214.464991:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574214.464995:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574214.464997:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574214.784932:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574214.784938:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574214.784942:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574214.784947:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574214.784951:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574214.784955:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574214.784970:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574214.784973:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574214.784976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574215.809011:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574215.809020:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574215.809027:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574215.809039:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574215.809050:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574215.809056:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574215.809065:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574215.809070:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574215.809076:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574216.832929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574216.832936:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574216.832939:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574216.832943:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574216.832950:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574216.832955:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574216.832960:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574216.833060:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574216.833064:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574217.088941:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309576941, at 4309576960 00000100:00000010:0.0:1788574217.088972:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea77e0. 00000020:00000001:0.0:1788574217.536989:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574217.536994:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574217.536998:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574217.856943:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574217.856949:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574217.856952:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574217.856957:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574217.856963:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574217.856968:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574217.856974:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574217.856976:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574217.856979:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574218.880892:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574218.880899:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574218.880904:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574218.880911:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574218.880919:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574218.880925:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574218.881088:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574218.881092:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574218.881096:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574219.584983:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574219.584988:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574219.584992:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574219.904898:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574219.904907:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574219.904911:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574219.904919:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574219.904925:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574219.904930:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574219.904935:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574219.904939:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574219.904943:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574219.905009:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574219.905013:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574219.905016:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574219.905020:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574219.905025:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574219.905029:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574219.905031:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574219.905032:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574219.905034:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574220.928912:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574220.928918:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574220.928923:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574220.928932:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574220.928937:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574220.928952:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574220.928959:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574220.928966:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574220.928971:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574221.952893:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574221.952899:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574221.952903:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574221.952911:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574221.952920:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574221.952926:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574221.952945:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574221.952950:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574221.952954:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574222.208887:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309582061, at 4309582080 00000100:00000010:0.0:1788574222.208908:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea77e0. 00000020:00000001:0.0:1788574222.656983:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574222.656987:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574222.656990:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574222.976917:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574222.976924:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574222.976927:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574222.976933:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574222.976940:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574222.976952:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574222.976977:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574222.976986:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574222.976991:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574224.000886:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574224.000895:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574224.000901:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574224.000912:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574224.000920:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574224.000926:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574224.000962:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574224.000966:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574224.000972:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574224.705071:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574224.705076:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574224.705079:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574225.024982:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574225.024995:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574225.025001:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574225.025013:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574225.025025:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574225.025033:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574225.025045:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574225.025051:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574225.025056:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574225.537105:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788574226.048958:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574226.048964:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574226.048968:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574226.048974:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574226.048985:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574226.048991:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574226.049019:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574226.049030:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574226.049039:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574227.072934:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574227.072940:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574227.072943:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574227.072977:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574227.072982:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574227.072987:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574227.072998:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574227.073002:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574227.073005:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574227.328913:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309587181, at 4309587200 00000100:00000010:0.0:1788574227.328929:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea79c0. 00000020:00000001:0.0:1788574227.777013:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574227.777017:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574227.777021:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574228.096882:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574228.096889:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574228.096893:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574228.096900:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574228.096907:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574228.096913:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574228.096923:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574228.096930:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574228.096937:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574228.608894:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574228.608906:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574228.608909:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574228.608914:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574228.608919:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574228.608927:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14916 00000400:00000200:0.0:1788574228.608946:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574228.608952:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574228.608955:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574228.608957:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.608960:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574228.608962:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.608964:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574228.608967:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574228.608984:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574228.608986:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574228.608988:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574228.608990:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574228.608992:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574228.608995:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.608996:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574228.608998:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.608999:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574228.609001:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574228.609004:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609006:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574228.609008:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574228.609011:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574228.609013:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.609014:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574228.609016:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574228.609021:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574228.609023:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574228.609026:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574228.609027:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.609030:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574228.609032:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574228.609036:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574228.609042:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574228.609048:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574228.609055:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516671232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574228.609063:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574228.609067:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574228.609073:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516671232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574228.609107:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574228.609111:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574228.609115:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574228.609117:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574228.609119:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574228.609122:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574228.609127:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14916 00000400:00000200:0.0:1788574228.609132:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574228.609139:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574228.609142:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574228.609146:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609152:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574228.609156:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609161:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574228.609164:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574228.609167:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574228.609170:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574228.609172:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574228.609175:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574228.609180:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574228.609186:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609192:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574228.609196:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609197:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574228.609201:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574228.609205:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609209:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574228.609217:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574228.609223:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574228.609226:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.609229:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574228.609234:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574228.609237:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.609240:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574228.609246:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574228.609249:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574228.609253:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574228.609254:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.609255:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574228.609257:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574228.609258:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574228.609261:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574228.609266:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574228.609272:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516671360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574228.609280:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574228.609284:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574228.609289:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516671360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574228.609312:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574228.609317:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574228.609321:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574228.609323:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574228.609326:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574228.609328:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574228.609332:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14916 00000400:00000200:0.0:1788574228.609338:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574228.609341:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574228.609342:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574228.609352:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609354:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574228.609360:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609365:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574228.609370:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574228.609374:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574228.609377:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574228.609383:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574228.609389:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574228.609396:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574228.609407:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609412:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574228.609419:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609425:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574228.609430:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574228.609436:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609440:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574228.609442:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574228.609447:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574228.609452:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.609456:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574228.609460:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574228.609477:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.609483:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574228.609491:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574228.609503:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574228.609510:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574228.609514:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.609516:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574228.609519:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574228.609522:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574228.609530:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574228.609540:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574228.609547:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516671488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574228.609574:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574228.609580:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574228.609584:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516671488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574228.609595:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574228.609599:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574228.609605:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574228.609606:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574228.609608:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574228.609613:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574228.609616:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14916 00000400:00000200:0.0:1788574228.609620:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574228.609624:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574228.609626:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574228.609631:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609634:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574228.609638:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609643:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574228.609649:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574228.609651:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574228.609654:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574228.609657:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574228.609662:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574228.609669:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574228.609676:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609682:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574228.609697:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609703:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574228.609708:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574228.609717:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609736:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574228.609738:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574228.609741:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574228.609744:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.609748:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574228.609753:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574228.609759:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.609762:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574228.609766:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574228.609770:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574228.609775:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574228.609778:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.609781:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574228.609785:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574228.609789:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574228.609841:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574228.609850:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574228.609861:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516671616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574228.609881:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574228.609887:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574228.609892:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516671616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574228.609906:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574228.609910:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574228.609914:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574228.609915:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574228.609926:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574228.609930:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574228.609936:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14916 00000400:00000200:0.0:1788574228.609943:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574228.609948:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574228.609951:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574228.609956:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609959:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574228.609963:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609965:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574228.609969:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574228.609970:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574228.609973:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574228.609977:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574228.609981:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574228.609984:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574228.609989:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.609994:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574228.609997:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574228.610000:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574228.610003:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574228.610006:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574228.610012:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574228.610017:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574228.610020:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574228.610022:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.610027:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574228.610029:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574228.610032:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.610033:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574228.610039:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574228.610042:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574228.610045:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574228.610046:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574228.610048:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574228.610051:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574228.610054:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574228.610062:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574228.610068:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574228.610074:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516671744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574228.610093:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574228.610096:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574228.610101:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516671744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574228.610115:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574228.610120:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14926) 00000100:00000001:0.0:1788574229.120908:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574229.120915:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574229.120921:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574229.120929:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574229.120935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574229.120941:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574229.120968:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574229.120973:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574229.120982:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574229.825025:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574229.825028:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574229.825031:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574230.144886:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574230.144893:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574230.144898:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574230.144906:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574230.144913:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574230.144919:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574230.144941:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574230.144944:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574230.144947:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574231.168923:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574231.168931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574231.168935:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574231.168942:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574231.168949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574231.168955:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574231.168963:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574231.168966:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574231.168969:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574232.192919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574232.192925:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574232.192930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574232.192937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574232.192944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574232.192950:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574232.192958:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574232.192962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574232.192965:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574232.448888:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309592301, at 4309592320 00000100:00000010:0.0:1788574232.448902:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7a80. 00000020:00000001:0.0:1788574232.896922:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574232.896925:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574232.896928:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574233.216929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574233.216934:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574233.216938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574233.216946:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574233.216953:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574233.216956:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574233.216965:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574233.216968:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574233.216971:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574234.240918:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574234.240925:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574234.240950:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574234.240956:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574234.240961:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574234.240964:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574234.240967:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574234.240969:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574234.240973:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574234.945019:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574234.945023:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574234.945026:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574235.264911:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574235.264915:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574235.264919:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574235.264922:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574235.264927:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574235.264932:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574235.264935:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574235.264939:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574235.264942:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574235.264960:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574235.264964:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574235.264966:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574235.264968:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574235.264973:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574235.264976:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574235.264978:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574235.264979:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574235.264981:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574236.288897:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574236.288902:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574236.288905:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574236.288913:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574236.288921:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574236.288929:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574236.288935:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574236.288939:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574236.288949:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574237.312896:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574237.312923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574237.312927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574237.312934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574237.312938:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574237.312943:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574237.312952:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574237.312955:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574237.312958:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574237.568900:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309597421, at 4309597440 00000100:00000010:0.0:1788574237.568914:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7420. 00000020:00000001:0.0:1788574238.017026:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574238.017031:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574238.017034:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574238.336882:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574238.336889:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574238.336891:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574238.336897:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574238.336904:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574238.336910:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574238.336916:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574238.336922:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574238.336928:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574239.360905:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574239.360913:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574239.360917:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574239.360923:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574239.360929:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574239.360934:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574239.360959:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574239.360963:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574239.360969:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574240.064985:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574240.064988:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574240.064990:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574240.384932:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574240.384937:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574240.384941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574240.384965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574240.384969:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574240.384972:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574240.384975:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574240.384978:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574240.384982:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574241.408904:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574241.408910:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574241.408915:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574241.408923:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574241.408930:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574241.408935:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574241.408942:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574241.408947:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574241.408955:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574241.408985:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574241.408989:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574241.408991:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574241.408993:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574241.408996:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574241.408998:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574241.409000:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574241.409001:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574241.409002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574242.432930:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574242.432938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574242.432943:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574242.432953:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574242.432961:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574242.432966:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574242.432985:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574242.432991:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574242.432998:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574242.688893:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309602542, at 4309602560 00000100:00000010:0.0:1788574242.688909:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7c60. 00000020:00000001:0.0:1788574243.137069:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574243.137073:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574243.137092:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574243.456895:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574243.456903:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574243.456904:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574243.456910:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574243.456915:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574243.456919:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574243.456924:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574243.456927:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574243.456931:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574244.480917:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574244.480926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574244.480929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574244.480937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574244.480943:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574244.480950:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574244.480959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574244.480964:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574244.480971:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574244.481014:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574244.481020:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574244.481022:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574244.481025:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574244.481029:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574244.481032:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574244.481034:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574244.481035:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574244.481038:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574245.185000:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574245.185011:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574245.185013:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574245.504947:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574245.504955:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574245.504961:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574245.504972:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574245.504979:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574245.504984:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574245.504993:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574245.505022:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574245.505028:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574245.505062:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574245.505067:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574245.505069:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574245.505072:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574245.505074:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574245.505076:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574245.505081:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574245.505087:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574245.505092:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574246.528933:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574246.528941:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574246.528946:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574246.528953:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574246.528961:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574246.528967:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574246.528983:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574246.528987:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574246.528993:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574247.552891:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574247.552896:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574247.552900:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574247.552908:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574247.552916:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574247.552923:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574247.552943:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574247.552947:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574247.552951:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574247.808936:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309607661, at 4309607680 00000100:00000010:0.0:1788574247.808957:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7180. 00000020:00000001:0.0:1788574248.257023:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574248.257027:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574248.257029:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574248.576912:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574248.576919:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574248.576924:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574248.576945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574248.576951:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574248.576956:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574248.576972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574248.576975:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574248.576978:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574249.600907:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574249.600916:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574249.600921:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574249.600931:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574249.600940:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574249.600946:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574249.600968:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574249.600972:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574249.600975:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574250.304980:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574250.304983:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574250.304986:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574250.624920:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574250.624926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574250.624930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574250.624936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574250.624944:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574250.624949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574250.624956:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574250.624961:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574250.624966:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574250.624990:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574250.624995:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574250.624997:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574250.625000:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574250.625004:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574250.625006:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574250.625008:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574250.625009:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574250.625011:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574251.648906:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574251.648912:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574251.648931:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574251.648937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574251.648944:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574251.648954:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574251.648958:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574251.648962:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574251.648965:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574251.648986:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574251.648991:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574251.648993:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574251.648996:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574251.648999:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574251.649002:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574251.649004:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574251.649006:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574251.649009:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574252.672931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574252.672938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574252.672942:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574252.672950:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574252.672956:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574252.672961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574252.672968:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574252.672974:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574252.672980:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574252.672994:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574252.672997:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574252.672999:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574252.673002:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574252.673006:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574252.673011:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574252.673015:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574252.673017:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574252.673019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574252.928897:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309612781, at 4309612800 00000100:00000010:0.0:1788574252.928914:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7b40. 00000020:00000001:0.0:1788574253.376986:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574253.377005:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574253.377008:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574253.696914:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574253.696918:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574253.696922:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574253.696926:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574253.696930:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574253.696933:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574253.696947:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574253.696950:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574253.696955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574254.208916:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574254.208926:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574254.208929:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574254.208932:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574254.208936:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574254.208942:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14941 00000400:00000200:0.0:1788574254.208951:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574254.208956:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574254.208959:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574254.208961:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.208964:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574254.208966:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.208968:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574254.208971:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574254.208972:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574254.208974:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574254.208977:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574254.208978:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574254.208980:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574254.208982:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.208983:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574254.208999:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209000:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574254.209002:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574254.209005:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209007:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574254.209009:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574254.209011:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574254.209013:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209015:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574254.209016:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574254.209022:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574254.209024:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574254.209027:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574254.209028:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209030:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574254.209033:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574254.209036:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574254.209043:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574254.209049:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574254.209056:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516674432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574254.209064:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574254.209067:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574254.209071:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516674432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574254.209087:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574254.209089:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574254.209092:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574254.209093:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574254.209094:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574254.209096:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574254.209120:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14941 00000400:00000200:0.0:1788574254.209127:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574254.209132:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574254.209134:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574254.209140:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209144:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574254.209148:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209161:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574254.209167:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574254.209170:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574254.209173:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574254.209177:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574254.209180:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574254.209184:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574254.209189:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209192:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574254.209195:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209200:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574254.209203:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574254.209208:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209213:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574254.209215:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574254.209224:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574254.209227:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209231:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574254.209235:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574254.209237:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209243:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574254.209251:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574254.209255:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574254.209260:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574254.209267:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209269:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574254.209272:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574254.209273:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574254.209278:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574254.209283:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574254.209289:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516674560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574254.209301:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574254.209304:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574254.209312:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516674560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574254.209322:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574254.209324:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574254.209328:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574254.209331:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574254.209334:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574254.209338:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574254.209344:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14941 00000400:00000200:0.0:1788574254.209352:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574254.209358:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574254.209361:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574254.209363:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209367:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574254.209371:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209373:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574254.209376:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574254.209379:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574254.209382:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574254.209384:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574254.209387:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574254.209398:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574254.209405:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209408:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574254.209413:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209418:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574254.209422:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574254.209424:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209430:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574254.209434:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574254.209437:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574254.209440:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209444:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574254.209447:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574254.209450:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209451:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574254.209455:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574254.209461:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574254.209462:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574254.209463:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209465:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574254.209468:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574254.209469:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574254.209474:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574254.209478:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574254.209485:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516674688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574254.209495:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574254.209498:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574254.209503:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516674688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574254.209516:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574254.209526:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574254.209532:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574254.209534:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574254.209536:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574254.209542:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574254.209545:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14941 00000400:00000200:0.0:1788574254.209550:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574254.209553:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574254.209555:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574254.209560:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209563:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574254.209569:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209573:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574254.209577:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574254.209579:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574254.209581:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574254.209585:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574254.209590:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574254.209595:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574254.209603:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209606:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574254.209610:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209613:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574254.209615:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574254.209616:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209619:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574254.209621:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574254.209623:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574254.209626:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209632:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574254.209639:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574254.209643:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209645:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574254.209650:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574254.209652:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574254.209655:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574254.209658:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209660:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574254.209663:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574254.209666:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574254.209670:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574254.209676:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574254.209684:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516674816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574254.209692:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574254.209695:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574254.209701:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516674816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574254.209726:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574254.209729:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574254.209736:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574254.209740:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574254.209744:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574254.209753:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574254.209761:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14941 00000400:00000200:0.0:1788574254.209773:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574254.209781:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574254.209787:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574254.209796:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209803:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574254.209851:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209864:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574254.209872:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574254.209875:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574254.209879:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574254.209882:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574254.209887:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574254.209891:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574254.209899:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209902:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574254.209907:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209910:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574254.209912:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574254.209918:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574254.209922:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574254.209926:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574254.209930:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574254.209934:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209938:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574254.209940:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574254.209944:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209946:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574254.209950:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574254.209953:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574254.209958:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574254.209959:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574254.209972:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574254.209974:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574254.209978:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574254.209983:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574254.209988:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574254.209993:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516674944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574254.210019:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574254.210023:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574254.210031:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516674944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574254.210059:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574254.210067:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14951) 00000100:00000001:0.0:1788574254.720907:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574254.720915:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574254.720920:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574254.720926:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574254.720933:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574254.720938:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574254.720951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574254.720956:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574254.720958:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574255.424998:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574255.425002:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574255.425006:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574256.768943:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574256.768958:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574256.768969:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574256.768976:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574256.768983:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574256.768986:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574256.769000:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574256.769003:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574256.769006:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574257.792891:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574257.792898:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574257.792902:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574257.792911:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574257.792921:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574257.792927:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574257.792948:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574257.792965:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574257.792968:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574258.048896:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309617901, at 4309617920 00000100:00000010:0.0:1788574258.048911:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ea0. 00000020:00000001:0.0:1788574258.496993:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574258.496996:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574258.496999:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574258.816910:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574258.816917:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574258.816922:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574258.816928:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574258.816934:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574258.816941:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574258.816948:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574258.816953:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574258.816959:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574258.816993:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574258.816998:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574258.817004:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574258.817010:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574258.817026:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574258.817028:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574258.817030:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574258.817031:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574258.817033:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574259.328935:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574259.328948:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574259.328951:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574259.328955:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574259.328961:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574259.328968:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14946 00000400:00000200:0.0:1788574259.328977:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574259.328983:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574259.328987:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574259.328990:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329023:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574259.329026:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329029:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574259.329033:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574259.329035:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574259.329038:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574259.329041:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574259.329044:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574259.329047:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574259.329050:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329052:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574259.329055:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329057:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574259.329060:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574259.329063:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329066:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574259.329082:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574259.329087:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574259.329090:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.329092:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574259.329095:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574259.329103:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574259.329106:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574259.329111:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574259.329112:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.329115:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574259.329118:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574259.329123:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574259.329131:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574259.329139:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574259.329157:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516675072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574259.329182:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574259.329186:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574259.329192:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516675072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574259.329210:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574259.329213:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574259.329215:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574259.329216:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574259.329218:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574259.329219:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574259.329221:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14946 00000400:00000200:0.0:1788574259.329225:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574259.329228:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574259.329230:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574259.329232:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329234:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574259.329236:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329238:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574259.329239:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574259.329241:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574259.329242:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574259.329244:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574259.329248:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574259.329250:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574259.329255:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329257:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574259.329262:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329267:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574259.329283:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574259.329292:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329297:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574259.329302:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574259.329311:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574259.329317:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.329322:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574259.329330:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574259.329336:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.329341:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574259.329349:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574259.329356:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574259.329362:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574259.329366:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.329369:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574259.329371:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574259.329376:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574259.329384:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574259.329393:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574259.329406:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516675200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574259.329419:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574259.329425:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574259.329434:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516675200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574259.329453:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574259.329459:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574259.329468:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574259.329472:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574259.329476:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574259.329482:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574259.329496:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14946 00000400:00000200:0.0:1788574259.329509:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574259.329517:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574259.329522:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574259.329528:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329534:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574259.329537:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329541:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574259.329545:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574259.329548:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574259.329551:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574259.329557:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574259.329563:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574259.329569:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574259.329574:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329580:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574259.329586:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329591:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574259.329598:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574259.329605:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329611:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574259.329616:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574259.329621:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574259.329627:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.329633:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574259.329638:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574259.329645:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.329650:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574259.329666:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574259.329672:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574259.329678:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574259.329690:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.329695:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574259.329699:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574259.329703:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574259.329711:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574259.329719:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574259.329730:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516675328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574259.329749:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574259.329756:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574259.329766:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516675328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574259.329791:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574259.329800:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574259.329806:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574259.329846:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574259.329851:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574259.329861:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574259.329870:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14946 00000400:00000200:0.0:1788574259.329883:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574259.329892:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574259.329898:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574259.329907:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329911:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574259.329915:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329919:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574259.329925:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574259.329927:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574259.329932:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574259.329937:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574259.329944:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574259.329956:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574259.329964:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329967:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574259.329969:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329972:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574259.329976:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574259.329982:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574259.329989:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574259.329992:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574259.329996:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574259.330000:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.330003:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574259.330008:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574259.330015:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.330021:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574259.330026:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574259.330031:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574259.330037:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574259.330041:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.330046:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574259.330051:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574259.330058:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574259.330067:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574259.330089:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574259.330099:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516675456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574259.330114:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574259.330119:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574259.330124:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516675456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574259.330141:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574259.330150:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574259.330159:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574259.330163:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574259.330167:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574259.330174:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574259.330179:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14946 00000400:00000200:0.0:1788574259.330188:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574259.330193:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574259.330195:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574259.330197:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.330203:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574259.330208:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.330214:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574259.330218:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574259.330222:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574259.330226:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574259.330229:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574259.330234:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574259.330238:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574259.330246:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.330250:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574259.330255:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574259.330260:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574259.330263:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574259.330270:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574259.330274:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574259.330277:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574259.330283:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574259.330290:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.330295:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574259.330305:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574259.330311:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.330315:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574259.330321:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574259.330326:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574259.330330:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574259.330334:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574259.330339:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574259.330341:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574259.330345:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574259.330349:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574259.330354:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574259.330364:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516675584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574259.330377:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574259.330380:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574259.330389:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516675584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574259.330400:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574259.330403:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14956) 00000100:00000001:0.0:1788574259.840982:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574259.840990:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574259.840994:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574259.841003:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574259.841009:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574259.841019:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574259.841044:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574259.841053:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574259.841058:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574260.545070:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574260.545073:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574260.545076:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574260.864947:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574260.864957:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574260.864988:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574260.864999:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574260.865009:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574260.865014:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574260.865025:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574260.865027:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574260.865032:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574261.888992:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574261.889003:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574261.889010:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574261.889028:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574261.889061:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574261.889069:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574261.889076:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574261.889079:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574261.889085:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574262.912895:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574262.912903:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574262.912908:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574262.912917:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574262.912925:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574262.912931:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574262.912943:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574262.912946:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574262.912949:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574263.168901:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309623021, at 4309623040 00000100:00000010:0.0:1788574263.168917:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7000. 00000020:00000001:0.0:1788574263.617057:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574263.617060:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574263.617063:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574263.936977:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574263.936991:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574263.937012:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574263.937024:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574263.937037:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574263.937043:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574263.937067:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574263.937101:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574263.937106:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574264.449128:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788574264.960896:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574264.960902:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574264.960906:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574264.960910:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574264.960917:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574264.960922:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574264.960932:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574264.960937:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574264.960940:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574264.960971:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574264.960974:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574264.960975:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574264.960977:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574264.960980:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574264.960997:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574264.960999:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574264.961001:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574264.961003:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574265.665008:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574265.665019:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574265.665023:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574265.984944:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574265.984949:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574265.984953:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574265.984960:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574265.984964:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574265.984984:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574265.984987:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574265.984992:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574265.984997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574267.008931:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574267.008939:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574267.008943:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574267.008962:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574267.008970:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574267.008982:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574267.008988:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574267.008993:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574267.008997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574268.032961:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574268.032968:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574268.032973:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574268.032980:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574268.032987:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574268.032992:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574268.033004:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574268.033007:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574268.033011:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574268.288899:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309628141, at 4309628160 00000100:00000010:0.0:1788574268.288913:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab240. 00000020:00000001:0.0:1788574268.737104:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574268.737109:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574268.737118:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574269.056951:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574269.056958:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574269.056963:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574269.056968:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574269.056975:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574269.056981:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574269.056992:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574269.056994:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574269.056996:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574270.080946:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574270.080953:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574270.080958:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574270.080965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574270.080970:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574270.080975:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574270.080987:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574270.080989:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574270.080992:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574270.785077:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574270.785088:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574270.785093:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574271.104924:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574271.104929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574271.104933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574271.104939:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574271.104946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574271.104951:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574271.104963:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574271.104966:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574271.104969:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574272.128990:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574272.128996:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574272.129000:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574272.129005:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574272.129008:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574272.129013:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574272.129018:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574272.129020:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574272.129023:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574273.152979:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574273.152996:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574273.153005:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574273.153022:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574273.153035:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574273.153042:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574273.153092:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574273.153096:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574273.153099:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574273.408903:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309633261, at 4309633280 00000100:00000010:0.0:1788574273.408917:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab960. 00000020:00000001:0.0:1788574273.857037:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574273.857041:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574273.857043:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574275.200910:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574275.200918:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574275.200922:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574275.200949:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574275.200955:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574275.200960:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574275.200965:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574275.200969:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574275.200973:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574275.905014:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574275.905018:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574275.905021:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574276.224938:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574276.224952:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574276.224960:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574276.224966:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574276.224975:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574276.224980:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574276.224985:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574276.224989:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574276.224994:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574277.248925:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574277.248930:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574277.248933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574277.248936:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574277.248939:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574277.248941:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574277.248943:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574277.248945:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574277.248948:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574278.272941:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574278.272947:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574278.272952:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574278.272960:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574278.272965:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574278.272969:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574278.272974:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574278.272979:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574278.272986:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574278.528915:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309638381, at 4309638400 00000100:00000010:0.0:1788574278.528932:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7f00. 00000020:00000001:0.0:1788574278.977056:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574278.977059:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574278.977062:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574279.296910:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574279.296914:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574279.296916:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574279.296919:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574279.296921:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574279.296925:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574279.296929:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574279.296932:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574279.296938:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574280.320888:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574280.320894:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574280.320899:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574280.320908:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574280.320914:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574280.320919:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574280.320937:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574280.320940:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574280.320943:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574280.320989:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574280.320992:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574280.320995:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574280.321000:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574280.321005:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574280.321010:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574280.321016:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574280.321019:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574280.321024:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574281.024980:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574281.024984:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574281.024987:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574281.344923:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574281.344931:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574281.344934:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574281.344940:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574281.344946:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574281.344970:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574281.344995:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574281.344998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574281.345002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574282.368922:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574282.368928:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574282.368933:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574282.368941:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574282.368947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574282.368951:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574282.368958:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574282.368960:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574282.368965:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574283.392914:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574283.392919:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574283.392923:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574283.392929:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574283.392933:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574283.392940:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574283.392954:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574283.392958:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574283.392962:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574283.648901:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309643501, at 4309643520 00000100:00000010:0.0:1788574283.648915:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7de0. 00000020:00000001:0.0:1788574284.097004:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574284.097007:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574284.097010:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574284.416907:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574284.416914:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574284.416916:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574284.416921:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574284.416926:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574284.416931:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574284.416936:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574284.416941:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574284.416948:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574284.416983:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574284.417006:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574284.417014:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574284.417029:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574284.417035:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574284.417040:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574284.417043:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574284.417045:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574284.417047:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574285.440925:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574285.440930:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574285.440933:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574285.440940:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574285.440947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574285.440952:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574285.440966:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574285.440969:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574285.440972:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574286.144980:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574286.144983:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574286.144986:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574286.464947:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574286.464959:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574286.464968:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574286.464981:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574286.464989:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574286.464993:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574286.464998:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574286.465002:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574286.465006:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574286.465040:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574286.465045:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574286.465051:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574286.465061:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574286.465082:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574286.465092:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574286.465107:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574286.465109:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574286.465113:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574287.488957:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574287.488967:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574287.488972:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574287.488983:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574287.488993:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574287.489000:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574287.489016:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574287.489021:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574287.489026:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574287.489055:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574287.489064:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574287.489072:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574287.489083:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574287.489096:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574287.489102:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574287.489107:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574287.489109:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574287.489113:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574288.512924:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574288.512933:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574288.512940:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574288.512952:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574288.512962:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574288.512980:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574288.513003:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574288.513007:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574288.513011:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574288.768912:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309648621, at 4309648640 00000100:00000010:0.0:1788574288.768926:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7900. 00000020:00000001:0.0:1788574289.217024:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574289.217029:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574289.217032:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574289.536950:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574289.536959:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574289.536963:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574289.536970:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574289.536977:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574289.537003:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574289.537010:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574289.537016:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574289.537027:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574290.560966:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574290.560977:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574290.560984:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574290.560996:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574290.561008:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574290.561019:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574290.561032:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574290.561043:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574290.561051:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574290.561109:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574290.561113:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574290.561114:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574290.561116:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574290.561118:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574290.561120:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574290.561122:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574290.561124:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574290.561125:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574291.264958:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574291.264963:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574291.264966:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574291.584997:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574291.585002:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574291.585006:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574291.585017:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574291.585021:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574291.585025:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574291.585031:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574291.585038:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574291.585042:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574292.608918:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574292.608926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574292.608931:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574292.608940:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574292.608967:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574292.608974:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574292.608977:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574292.608980:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574292.608983:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574293.632970:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574293.632981:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574293.632987:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574293.633005:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574293.633020:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574293.633029:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574293.633047:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574293.633051:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574293.633057:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574293.888921:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309653741, at 4309653760 00000100:00000010:0.0:1788574293.888936:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea71e0. 00000020:00000001:0.0:1788574294.337105:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574294.337111:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574294.337115:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574294.657070:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574294.657074:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574294.657077:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574294.657080:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574294.657085:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574294.657087:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574294.657089:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574294.657091:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574294.657093:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574295.680973:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574295.680980:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574295.680984:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574295.680998:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574295.681009:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574295.681015:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574295.681027:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574295.681038:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574295.681059:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574296.384979:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574296.384985:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574296.384991:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574296.704938:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574296.704950:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574296.704956:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574296.704962:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574296.704969:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574296.704974:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574296.704998:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574296.705002:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574296.705006:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574297.728921:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574297.728926:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574297.728931:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574297.728937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574297.728943:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574297.728948:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574297.728958:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574297.728962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574297.728967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574298.752910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574298.752916:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574298.752921:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574298.752931:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574298.752939:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574298.752945:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574298.752949:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574298.752953:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574298.752959:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574299.008891:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309658861, at 4309658880 00000100:00000010:0.0:1788574299.008905:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea78a0. 00000020:00000001:0.0:1788574299.456951:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574299.456954:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574299.456956:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574299.776970:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574299.776973:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574299.776981:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574299.776986:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574299.776991:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574299.776996:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574299.777010:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574299.777014:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574299.777017:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574300.288894:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574300.288902:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574300.288904:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574300.288915:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574300.288920:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574300.288926:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 14987 00000400:00000200:0.0:1788574300.288934:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574300.288940:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574300.288943:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574300.288945:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.288948:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574300.288949:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.288951:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574300.288955:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574300.288956:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574300.288958:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574300.288961:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574300.288962:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574300.288964:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574300.288966:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.288968:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574300.288969:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.288971:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574300.288973:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574300.288975:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574300.288992:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574300.288995:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574300.288997:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574300.288999:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.289001:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574300.289003:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574300.289014:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574300.289017:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574300.289020:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574300.289021:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.289023:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574300.289026:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574300.289030:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574300.289036:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574300.289041:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574300.289048:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516680192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574300.289055:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574300.289059:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574300.289062:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516680192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574300.289077:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574300.289079:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574300.289082:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574300.289087:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574300.289088:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574300.289090:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574300.289093:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14987 00000400:00000200:0.0:1788574300.289097:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574300.289099:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574300.289101:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574300.289104:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289113:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574300.289115:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289117:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574300.289118:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574300.289120:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574300.289124:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574300.289126:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574300.289134:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574300.289140:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574300.289147:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289150:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574300.289154:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289159:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574300.289165:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574300.289173:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289178:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574300.289183:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574300.289192:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574300.289200:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.289204:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574300.289211:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574300.289215:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.289217:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574300.289222:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574300.289226:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574300.289231:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574300.289234:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.289238:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574300.289241:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574300.289244:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574300.289250:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574300.289263:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574300.289277:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516680320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574300.289292:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574300.289296:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574300.289304:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516680320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574300.289327:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574300.289329:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574300.289334:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574300.289338:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574300.289341:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574300.289345:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574300.289351:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14987 00000400:00000200:0.0:1788574300.289357:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574300.289361:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574300.289363:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574300.289370:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289375:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574300.289378:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289381:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574300.289384:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574300.289387:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574300.289390:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574300.289392:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574300.289393:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574300.289395:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574300.289398:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289399:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574300.289400:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289405:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574300.289408:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574300.289414:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289419:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574300.289421:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574300.289423:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574300.289429:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.289431:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574300.289434:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574300.289438:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.289441:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574300.289445:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574300.289448:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574300.289450:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574300.289451:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.289455:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574300.289457:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574300.289462:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574300.289479:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574300.289495:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574300.289507:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516680448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574300.289529:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574300.289533:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574300.289539:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516680448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574300.289558:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574300.289560:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574300.289567:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574300.289570:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574300.289572:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574300.289585:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574300.289590:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14987 00000400:00000200:0.0:1788574300.289597:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574300.289607:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574300.289609:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574300.289615:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289619:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574300.289623:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289626:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574300.289628:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574300.289629:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574300.289632:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574300.289634:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574300.289638:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574300.289643:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574300.289649:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289652:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574300.289657:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289662:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574300.289665:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574300.289670:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289673:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574300.289676:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574300.289680:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574300.289685:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.289689:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574300.289693:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574300.289702:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.289704:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574300.289708:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574300.289718:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574300.289725:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574300.289729:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.289734:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574300.289738:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574300.289742:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574300.289750:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574300.289757:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574300.289769:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516680576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574300.289786:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574300.289788:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574300.289792:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516680576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574300.289842:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574300.289845:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574300.289852:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574300.289857:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574300.289860:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574300.289869:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574300.289876:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 14987 00000400:00000200:0.0:1788574300.289885:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574300.289893:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574300.289900:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574300.289906:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289911:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574300.289916:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289923:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574300.289928:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574300.289932:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574300.289943:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574300.289948:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574300.289955:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574300.289961:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574300.289970:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289974:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574300.289981:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574300.289986:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574300.289992:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574300.289999:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574300.290006:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574300.290012:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574300.290019:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574300.290024:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.290029:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574300.290034:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574300.290041:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.290045:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574300.290052:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574300.290058:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574300.290064:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574300.290068:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574300.290073:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574300.290077:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574300.290082:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574300.290089:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574300.290096:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574300.290106:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516680704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574300.290127:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574300.290133:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574300.290144:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516680704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574300.290177:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574300.290185:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (14997) 00000100:00000001:0.0:1788574300.800915:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574300.800923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574300.800928:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574300.800936:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574300.800945:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574300.800950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574300.800953:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574300.800957:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574300.800962:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574301.504980:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574301.504985:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574301.504989:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574301.824904:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574301.824911:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574301.824916:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574301.824923:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574301.824928:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574301.824933:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574301.824938:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574301.824941:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574301.824944:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574302.848913:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574302.848920:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574302.848923:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574302.848927:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574302.848935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574302.848939:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574302.848954:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574302.848957:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574302.848962:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574303.361039:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788574303.553009:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574303.553012:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574303.553015:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574303.872898:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574303.872905:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574303.872907:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574303.872916:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574303.872924:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574303.872930:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574303.872933:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574303.872938:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574303.872950:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574304.128989:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309663981, at 4309664000 00000100:00000010:0.0:1788574304.128993:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea71e0. 00000020:00000001:0.0:1788574304.576995:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574304.576999:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574304.577003:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574304.896924:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574304.896930:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574304.896934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574304.896940:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574304.896952:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574304.896956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574304.896972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574304.896975:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574304.896979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574305.920891:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574305.920897:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574305.920901:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574305.920905:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574305.920911:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574305.920918:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574305.920944:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574305.920946:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574305.920948:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574306.624957:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574306.624961:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574306.624964:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574307.968931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574307.968936:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574307.968955:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574307.968962:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574307.968968:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574307.968972:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574307.968977:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574307.968980:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574307.968982:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574308.672971:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574308.672974:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574308.672977:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574308.992920:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574308.992927:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574308.992931:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574308.992940:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574308.992946:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574308.992951:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574308.992954:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574308.992956:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574308.992961:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574309.248905:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309669102, at 4309669120 00000100:00000010:0.0:1788574309.248920:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7900. 00000020:00000001:0.0:1788574309.696989:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574309.696992:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574309.696995:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574310.016922:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574310.016928:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574310.016932:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574310.016941:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574310.016957:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574310.016966:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574310.016988:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574310.016991:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574310.016995:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574311.040914:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574311.040918:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574311.040923:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574311.040930:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574311.040955:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574311.040961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574311.040966:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574311.040968:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574311.040971:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574311.745015:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574311.745019:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574311.745022:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574312.064929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574312.064937:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574312.064941:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574312.064947:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574312.064954:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574312.064959:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574312.064964:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574312.064969:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574312.064974:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574313.088937:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574313.088949:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574313.088956:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574313.088969:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574313.088976:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574313.088982:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574313.088988:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574313.088993:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574313.089001:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574313.793031:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574313.793037:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574313.793041:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574314.112948:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574314.112958:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574314.112965:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574314.112973:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574314.112986:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574314.112993:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574314.113010:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574314.113015:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574314.113019:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574314.368952:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309674221, at 4309674240 00000100:00000010:0.0:1788574314.368967:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7840. 00000020:00000001:0.0:1788574314.816932:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574314.816936:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574314.816940:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574315.137087:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574315.137091:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574315.137092:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574315.137094:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574315.137096:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574315.137097:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574315.137099:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574315.137100:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574315.137101:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574315.137110:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574315.137113:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574315.137116:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574315.137120:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574315.137125:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574315.137128:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574315.137132:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574315.137135:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574315.137137:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574316.160899:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574316.160905:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574316.160907:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574316.160911:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574316.160916:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574316.160920:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574316.160924:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574316.160927:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574316.160933:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574316.864969:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574316.864973:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574316.864977:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574317.184967:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574317.184973:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574317.184983:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574317.184987:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574317.184993:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574317.184997:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574317.185010:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574317.185013:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574317.185015:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574317.185032:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574317.185034:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574317.185035:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574317.185037:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574317.185039:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574317.185041:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574317.185043:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574317.185044:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574317.185046:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574318.208933:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574318.208938:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574318.208944:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574318.208952:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574318.208960:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574318.208965:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574318.208980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574318.208985:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574318.208989:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574318.913029:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574318.913033:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574318.913035:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574319.232925:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574319.232929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574319.232933:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574319.232942:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574319.232960:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574319.232964:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574319.232974:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574319.232977:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574319.232980:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574319.232996:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574319.233013:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574319.233015:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574319.233019:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574319.233022:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574319.233025:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574319.233028:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574319.233030:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574319.233033:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574319.488891:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309679341, at 4309679360 00000100:00000010:0.0:1788574319.488905:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7d80. 00000020:00000001:0.0:1788574319.936990:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574319.936993:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574319.936996:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574320.256924:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574320.256930:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574320.256935:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574320.256944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574320.256950:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574320.256954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574320.256969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574320.256972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574320.256974:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574320.256991:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574320.256994:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574320.256996:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574320.256998:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574320.257001:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574320.257003:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574320.257004:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574320.257006:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574320.257008:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574321.280935:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574321.280940:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574321.280945:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574321.280951:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574321.280954:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574321.280958:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574321.280980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574321.280984:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574321.280987:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574321.984989:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574321.984993:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574321.984997:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574322.304931:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574322.304936:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574322.304939:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574322.304945:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574322.304951:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574322.304956:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574322.304960:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574322.304963:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574322.304965:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574323.328929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574323.328932:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574323.328934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574323.328936:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574323.328938:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574323.328942:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574323.328953:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574323.328958:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574323.328970:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574324.033019:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574324.033022:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574324.033025:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574324.352926:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574324.352931:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574324.352934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574324.352937:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574324.352943:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574324.352949:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574324.352952:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574324.352958:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574324.352962:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574324.608930:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309684461, at 4309684480 00000100:00000010:0.0:1788574324.608946:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7000. 00000020:00000001:0.0:1788574325.057002:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574325.057005:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574325.057008:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574325.376973:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574325.376979:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574325.376982:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574325.376988:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574325.376993:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574325.376997:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574325.376999:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574325.377002:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574325.377005:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574325.888943:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574325.888955:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574325.888958:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574325.888962:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574325.888967:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574325.888973:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15013 00000400:00000200:0.0:1788574325.888983:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574325.888989:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574325.888993:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574325.888996:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889000:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574325.889003:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889005:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574325.889009:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574325.889011:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574325.889015:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574325.889018:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574325.889021:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574325.889024:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574325.889027:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889030:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574325.889050:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889052:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574325.889056:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574325.889060:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889062:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574325.889066:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574325.889069:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574325.889073:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.889076:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574325.889078:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574325.889086:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574325.889091:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574325.889095:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574325.889097:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.889100:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574325.889103:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574325.889107:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574325.889115:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574325.889122:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574325.889133:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516683392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574325.889144:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574325.889149:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574325.889154:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516683392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574325.889178:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574325.889181:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574325.889185:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574325.889187:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574325.889189:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574325.889192:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574325.889201:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15013 00000400:00000200:0.0:1788574325.889208:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574325.889213:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574325.889217:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574325.889224:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889231:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574325.889236:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889241:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574325.889246:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574325.889249:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574325.889253:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574325.889258:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574325.889265:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574325.889271:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574325.889280:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889285:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574325.889291:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889298:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574325.889303:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574325.889309:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889313:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574325.889316:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574325.889325:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574325.889332:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.889336:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574325.889343:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574325.889348:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.889353:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574325.889361:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574325.889368:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574325.889375:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574325.889385:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.889389:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574325.889393:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574325.889396:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574325.889405:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574325.889413:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574325.889423:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516683520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574325.889440:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574325.889445:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574325.889455:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516683520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574325.889481:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574325.889486:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574325.889494:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574325.889498:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574325.889502:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574325.889507:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574325.889512:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15013 00000400:00000200:0.0:1788574325.889520:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574325.889530:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574325.889533:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574325.889542:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889546:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574325.889553:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889559:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574325.889565:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574325.889568:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574325.889572:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574325.889576:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574325.889582:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574325.889594:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574325.889604:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889609:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574325.889614:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889617:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574325.889621:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574325.889627:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889632:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574325.889636:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574325.889641:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574325.889646:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.889649:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574325.889653:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574325.889656:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.889660:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574325.889666:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574325.889670:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574325.889673:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574325.889676:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.889679:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574325.889683:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574325.889686:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574325.889692:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574325.889698:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574325.889706:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516683648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574325.889722:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574325.889726:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574325.889735:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516683648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574325.889746:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574325.889772:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574325.889780:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574325.889784:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574325.889787:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574325.889790:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574325.889796:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15013 00000400:00000200:0.0:1788574325.889804:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574325.889838:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574325.889844:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574325.889849:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889853:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574325.889857:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889861:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574325.889865:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574325.889867:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574325.889870:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574325.889873:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574325.889877:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574325.889881:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574325.889885:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889888:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574325.889893:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889896:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574325.889900:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574325.889905:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574325.889909:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574325.889913:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574325.889916:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574325.889919:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.889923:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574325.889931:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574325.889935:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.889940:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574325.889942:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574325.889947:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574325.889952:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574325.889953:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.889956:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574325.889958:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574325.889960:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574325.889964:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574325.889970:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574325.889978:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516683776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574325.889987:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574325.889989:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574325.889995:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516683776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574325.890008:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574325.890012:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574325.890019:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574325.890021:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574325.890025:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574325.890030:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574325.890032:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15013 00000400:00000200:0.0:1788574325.890037:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574325.890042:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574325.890046:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574325.890051:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.890055:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574325.890060:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.890069:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574325.890073:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574325.890076:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574325.890078:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574325.890082:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574325.890086:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574325.890089:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574325.890094:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.890099:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574325.890103:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574325.890106:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574325.890108:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574325.890113:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574325.890118:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574325.890121:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574325.890126:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574325.890131:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.890134:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574325.890137:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574325.890141:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.890144:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574325.890150:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574325.890155:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574325.890159:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574325.890161:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574325.890166:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574325.890169:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574325.890173:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574325.890177:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574325.890183:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574325.890192:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516683904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574325.890211:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574325.890215:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574325.890222:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516683904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574325.890235:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574325.890240:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15023) 00000100:00000001:0.0:1788574326.400946:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574326.400953:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574326.400959:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574326.400965:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574326.400971:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574326.400976:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574326.400979:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574326.400981:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574326.400985:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574327.105054:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574327.105061:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574327.105065:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574327.424975:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574327.424984:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574327.424986:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574327.424993:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574327.425001:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574327.425007:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574327.425018:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574327.425025:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574327.425030:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574328.448899:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574328.448906:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574328.448909:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574328.448915:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574328.448919:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574328.448926:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574328.448933:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574328.448949:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574328.448955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574329.153084:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574329.153087:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574329.153089:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574329.472901:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574329.472909:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574329.472912:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574329.472918:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574329.472922:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574329.472927:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574329.472933:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574329.472937:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574329.472943:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574329.472974:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574329.472977:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574329.472980:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574329.472983:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574329.472988:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574329.472991:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574329.472995:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574329.472997:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574329.473001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574329.728879:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309689581, at 4309689600 00000100:00000010:0.0:1788574329.728892:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ea0. 00000020:00000001:0.0:1788574330.176996:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574330.176999:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574330.177001:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574330.496953:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574330.496959:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574330.496965:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574330.496970:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574330.496975:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574330.496977:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574330.496994:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574330.496997:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574330.497000:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574331.008911:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574331.008923:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574331.008926:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574331.008931:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574331.008936:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574331.008946:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15018 00000400:00000200:0.0:1788574331.008956:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574331.008962:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574331.008966:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574331.008971:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.008974:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574331.008978:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.008981:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574331.008985:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574331.008987:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574331.008990:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574331.008993:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574331.008996:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574331.008999:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574331.009002:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009005:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574331.009007:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009009:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574331.009013:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574331.009016:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009019:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574331.009022:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574331.009025:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574331.009029:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.009031:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574331.009033:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574331.009058:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574331.009062:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574331.009066:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574331.009068:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.009071:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574331.009074:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574331.009079:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574331.009087:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574331.009094:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574331.009114:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516684032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574331.009125:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574331.009129:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574331.009135:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516684032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574331.009163:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574331.009167:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574331.009171:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574331.009173:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574331.009176:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574331.009180:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574331.009183:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15018 00000400:00000200:0.0:1788574331.009189:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574331.009194:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574331.009199:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574331.009206:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009212:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574331.009217:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009222:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574331.009228:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574331.009230:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574331.009243:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574331.009247:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574331.009252:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574331.009257:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574331.009265:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009271:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574331.009278:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009284:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574331.009288:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574331.009294:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009296:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574331.009299:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574331.009306:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574331.009311:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.009315:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574331.009320:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574331.009327:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.009331:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574331.009338:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574331.009343:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574331.009348:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574331.009350:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.009355:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574331.009357:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574331.009360:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574331.009368:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574331.009376:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574331.009386:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516684160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574331.009406:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574331.009410:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574331.009419:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516684160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574331.009443:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574331.009449:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574331.009455:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574331.009458:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574331.009461:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574331.009467:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574331.009474:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15018 00000400:00000200:0.0:1788574331.009483:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574331.009488:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574331.009493:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574331.009501:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009506:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574331.009512:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009517:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574331.009522:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574331.009524:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574331.009528:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574331.009532:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574331.009536:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574331.009542:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574331.009550:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009554:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574331.009559:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009563:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574331.009568:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574331.009578:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009590:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574331.009601:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574331.009607:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574331.009614:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.009617:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574331.009620:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574331.009626:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.009632:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574331.009638:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574331.009644:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574331.009650:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574331.009653:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.009658:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574331.009661:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574331.009665:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574331.009672:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574331.009678:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574331.009689:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516684288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574331.009709:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574331.009712:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574331.009720:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516684288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574331.009738:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574331.009742:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574331.009749:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574331.009751:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574331.009755:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574331.009760:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574331.009765:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15018 00000400:00000200:0.0:1788574331.009773:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574331.009780:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574331.009785:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574331.009799:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009855:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574331.009860:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009867:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574331.009873:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574331.009877:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574331.009882:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574331.009886:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574331.009892:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574331.009898:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574331.009908:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009912:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574331.009918:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009926:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574331.009933:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574331.009940:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574331.009946:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574331.009951:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574331.009957:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574331.009979:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.009985:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574331.009990:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574331.009995:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.009999:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574331.010007:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574331.010013:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574331.010018:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574331.010022:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.010026:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574331.010028:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574331.010032:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574331.010039:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574331.010054:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574331.010065:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516684416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574331.010083:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574331.010087:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574331.010097:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516684416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574331.010114:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574331.010119:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574331.010128:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574331.010132:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574331.010135:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574331.010141:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574331.010152:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15018 00000400:00000200:0.0:1788574331.010161:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574331.010167:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574331.010170:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574331.010178:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.010183:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574331.010190:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.010195:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574331.010200:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574331.010203:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574331.010207:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574331.010211:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574331.010215:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574331.010222:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574331.010229:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.010231:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574331.010243:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574331.010246:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574331.010249:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574331.010252:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574331.010255:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574331.010257:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574331.010261:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574331.010263:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.010266:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574331.010268:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574331.010271:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.010273:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574331.010277:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574331.010281:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574331.010284:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574331.010286:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574331.010288:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574331.010289:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574331.010292:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574331.010296:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574331.010300:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574331.010306:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516684544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574331.010315:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574331.010317:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574331.010321:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516684544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574331.010332:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574331.010338:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15028) 00000100:00000001:0.0:1788574331.520908:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574331.520916:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574331.520920:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574331.520942:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574331.520960:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574331.520964:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574331.520967:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574331.520969:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574331.520971:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574332.225040:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574332.225045:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574332.225049:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574332.544934:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574332.544944:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574332.544948:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574332.544958:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574332.544966:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574332.544971:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574332.544978:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574332.544982:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574332.544984:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574333.569029:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574333.569034:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574333.569037:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574333.569040:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574333.569044:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574333.569047:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574333.569050:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574333.569052:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574333.569054:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574334.273034:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574334.273038:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574334.273040:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574334.592953:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574334.592963:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574334.592967:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574334.592982:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574334.592989:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574334.592995:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574334.593007:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574334.593011:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574334.593026:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574334.849011:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309694701, at 4309694720 00000100:00000010:0.0:1788574334.849016:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7360. 00000020:00000001:0.0:1788574335.297046:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574335.297049:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574335.297052:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574335.616959:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574335.616965:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574335.616969:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574335.616975:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574335.616980:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574335.616985:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574335.616989:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574335.616993:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574335.616996:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574336.128901:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574336.128912:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574336.128916:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574336.128920:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574336.128925:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574336.128934:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15023 00000400:00000200:0.0:1788574336.128944:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574336.128951:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574336.128955:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574336.128959:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.128962:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574336.128965:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.128967:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574336.128972:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574336.128974:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574336.128977:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574336.128980:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574336.128983:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574336.129002:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574336.129005:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129007:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574336.129008:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129009:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574336.129012:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574336.129015:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129017:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574336.129019:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574336.129022:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574336.129024:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.129025:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574336.129027:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574336.129031:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574336.129034:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574336.129037:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574336.129038:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.129041:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574336.129043:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574336.129046:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574336.129052:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574336.129057:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574336.129065:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516684672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574336.129073:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574336.129076:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574336.129079:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516684672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574336.129097:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574336.129099:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574336.129102:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574336.129107:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574336.129109:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574336.129112:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574336.129115:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15023 00000400:00000200:0.0:1788574336.129119:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574336.129122:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574336.129125:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574336.129131:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129134:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574336.129139:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129143:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574336.129146:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574336.129148:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574336.129151:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574336.129155:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574336.129158:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574336.129161:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574336.129165:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129168:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574336.129171:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129175:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574336.129179:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574336.129185:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129188:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574336.129191:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574336.129198:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574336.129201:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.129205:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574336.129211:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574336.129215:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.129222:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574336.129227:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574336.129230:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574336.129233:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574336.129235:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.129238:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574336.129241:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574336.129244:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574336.129249:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574336.129255:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574336.129261:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516684800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574336.129291:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574336.129295:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574336.129301:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516684800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574336.129314:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574336.129318:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574336.129323:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574336.129325:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574336.129328:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574336.129334:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574336.129340:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15023 00000400:00000200:0.0:1788574336.129347:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574336.129354:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574336.129359:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574336.129364:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129367:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574336.129372:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129376:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574336.129379:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574336.129388:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574336.129394:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574336.129400:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574336.129407:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574336.129413:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574336.129423:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129428:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574336.129436:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129439:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574336.129444:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574336.129448:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129452:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574336.129454:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574336.129460:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574336.129466:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.129470:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574336.129474:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574336.129481:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.129487:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574336.129493:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574336.129499:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574336.129503:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574336.129505:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.129506:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574336.129508:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574336.129511:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574336.129517:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574336.129524:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574336.129531:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516684928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574336.129544:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574336.129548:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574336.129560:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516684928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574336.129573:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574336.129578:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574336.129583:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574336.129587:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574336.129590:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574336.129592:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574336.129597:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15023 00000400:00000200:0.0:1788574336.129606:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574336.129612:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574336.129616:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574336.129621:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129625:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574336.129630:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129634:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574336.129638:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574336.129641:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574336.129644:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574336.129647:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574336.129651:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574336.129655:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574336.129663:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129667:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574336.129671:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129675:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574336.129681:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574336.129687:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129692:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574336.129702:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574336.129706:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574336.129709:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.129712:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574336.129716:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574336.129721:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.129726:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574336.129730:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574336.129734:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574336.129739:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574336.129741:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.129744:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574336.129748:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574336.129750:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574336.129755:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574336.129759:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574336.129767:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516685056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574336.129781:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574336.129784:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574336.129791:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516685056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574336.129834:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574336.129837:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574336.129843:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574336.129847:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574336.129849:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574336.129865:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574336.129869:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15023 00000400:00000200:0.0:1788574336.129875:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574336.129881:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574336.129884:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574336.129900:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129907:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574336.129914:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129921:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574336.129925:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574336.129929:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574336.129935:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574336.129940:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574336.129948:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574336.129954:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574336.129963:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129969:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574336.129975:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129982:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574336.129989:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574336.129994:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574336.129998:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574336.130001:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574336.130005:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574336.130008:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.130013:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574336.130015:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574336.130019:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.130021:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574336.130028:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574336.130033:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574336.130037:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574336.130039:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.130042:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574336.130044:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574336.130047:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574336.130053:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574336.130065:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574336.130075:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516685184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574336.130093:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574336.130097:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574336.130104:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516685184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574336.130120:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574336.130128:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15033) 00000100:00000001:0.0:1788574336.640930:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574336.640937:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574336.640941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574336.640947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574336.640953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574336.640958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574336.640972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574336.640978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574336.640986:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574336.641020:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574336.641025:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574336.641026:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574336.641029:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574336.641032:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574336.641035:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574336.641037:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574336.641039:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574336.641042:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574337.345098:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574337.345102:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574337.345105:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574338.688977:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574338.688982:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574338.688987:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574338.688995:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574338.689002:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574338.689025:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574338.689033:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574338.689037:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574338.689041:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574339.393192:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574339.393199:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574339.393204:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574339.713135:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574339.713144:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574339.713148:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574339.713154:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574339.713160:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574339.713165:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574339.713168:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574339.713171:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574339.713175:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574339.968956:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309699821, at 4309699840 00000100:00000010:0.0:1788574339.968980:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7600. 00000020:00000001:0.0:1788574340.417069:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574340.417075:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574340.417079:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574340.736962:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574340.736970:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574340.736975:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574340.736989:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574340.736998:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574340.737007:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574340.737012:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574340.737021:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574340.737025:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574341.248961:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574341.248978:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574341.248982:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574341.248998:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574341.249004:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574341.249011:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15028 00000400:00000200:0.0:1788574341.249035:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574341.249045:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574341.249050:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574341.249054:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249059:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574341.249062:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249066:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574341.249071:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574341.249074:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574341.249077:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574341.249081:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574341.249085:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574341.249089:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574341.249093:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249095:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574341.249098:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249100:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574341.249104:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574341.249108:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249111:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574341.249115:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574341.249119:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574341.249123:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.249126:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574341.249129:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574341.249141:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574341.249145:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574341.249150:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574341.249153:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.249156:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574341.249160:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574341.249167:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574341.249183:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574341.249192:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574341.249205:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516685312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574341.249219:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574341.249225:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574341.249230:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516685312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574341.249257:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574341.249262:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574341.249268:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574341.249272:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574341.249274:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574341.249279:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574341.249285:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15028 00000400:00000200:0.0:1788574341.249294:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574341.249303:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574341.249310:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574341.249319:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249326:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574341.249333:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249340:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574341.249348:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574341.249353:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574341.249358:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574341.249363:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574341.249369:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574341.249375:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574341.249381:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249404:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574341.249410:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249416:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574341.249419:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574341.249423:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249428:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574341.249430:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574341.249440:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574341.249446:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.249451:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574341.249459:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574341.249462:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.249463:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574341.249467:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574341.249472:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574341.249476:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574341.249479:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.249481:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574341.249484:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574341.249487:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574341.249495:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574341.249500:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574341.249509:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516685440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574341.249524:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574341.249528:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574341.249536:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516685440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574341.249559:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574341.249565:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574341.249579:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574341.249591:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574341.249594:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574341.249599:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574341.249607:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15028 00000400:00000200:0.0:1788574341.249618:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574341.249636:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574341.249642:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574341.249652:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249657:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574341.249664:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249670:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574341.249674:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574341.249677:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574341.249682:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574341.249687:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574341.249692:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574341.249698:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574341.249707:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249713:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574341.249719:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249726:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574341.249731:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574341.249735:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574341.249741:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574341.249743:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574341.249748:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574341.249754:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.249759:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574341.249764:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574341.249769:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.249785:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574341.249794:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574341.249801:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574341.249836:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574341.249839:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.249843:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574341.249849:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574341.249853:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574341.249863:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574341.249872:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574341.249884:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516685568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574341.249913:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574341.249920:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574341.249930:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516685568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574341.249947:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574341.249952:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574341.249957:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574341.249959:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574341.249963:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574341.249972:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574341.249980:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15028 00000400:00000200:0.0:1788574341.249993:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574341.250002:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574341.250008:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574341.250018:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.250024:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574341.250031:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.250039:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574341.250044:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574341.250053:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574341.250057:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574341.250061:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574341.250065:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574341.250068:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574341.250074:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.250079:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574341.250085:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.250088:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574341.250093:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574341.250099:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574341.250102:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574341.250106:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574341.250110:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574341.250112:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.250117:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574341.250120:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574341.250122:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.250124:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574341.250130:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574341.250135:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574341.250141:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574341.250144:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.250149:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574341.250151:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574341.250154:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574341.250161:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574341.250169:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574341.250182:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516685696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574341.250197:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574341.250200:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574341.250212:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516685696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574341.250224:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574341.250227:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574341.250232:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574341.250235:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574341.250238:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574341.250242:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574341.250245:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15028 00000400:00000200:0.0:1788574341.250253:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574341.250260:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574341.250263:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574341.250271:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.250278:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574341.250286:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.250293:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574341.250299:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574341.250304:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574341.250310:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574341.250316:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574341.250324:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574341.250332:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574341.250342:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.250349:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574341.250355:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574341.250362:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574341.250364:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574341.250370:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574341.250376:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574341.250386:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574341.250390:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574341.250395:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.250398:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574341.250402:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574341.250407:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.250412:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574341.250417:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574341.250422:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574341.250427:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574341.250430:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574341.250434:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574341.250438:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574341.250441:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574341.250447:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574341.250454:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574341.250465:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516685824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574341.250489:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574341.250494:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574341.250500:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516685824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574341.250516:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574341.250520:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15038) 00000100:00000001:0.0:1788574341.760989:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574341.761000:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574341.761004:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574341.761012:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574341.761021:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574341.761027:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574341.761047:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574341.761050:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574341.761054:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574342.465014:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574342.465036:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574342.465039:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574342.785095:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574342.785098:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574342.785099:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574342.785102:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574342.785103:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574342.785105:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574342.785107:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574342.785108:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574342.785110:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574343.809013:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574343.809019:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574343.809022:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574343.809027:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574343.809033:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574343.809038:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574343.809043:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574343.809047:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574343.809051:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574344.513018:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574344.513022:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574344.513025:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574344.833046:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574344.833052:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574344.833054:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574344.833058:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574344.833061:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574344.833064:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574344.833066:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574344.833068:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574344.833071:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574345.088899:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309704941, at 4309704960 00000100:00000010:0.0:1788574345.088917:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc485ea7ba0. 00000020:00000001:0.0:1788574345.536999:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574345.537002:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574345.537005:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574345.856919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574345.856928:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574345.856934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574345.856941:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574345.856962:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574345.856968:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574345.856977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574345.856981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574345.856986:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574345.857024:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574345.857028:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574345.857030:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574345.857033:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574345.857036:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574345.857039:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574345.857042:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574345.857043:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574345.857046:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574346.880929:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574346.880935:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574346.880939:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574346.880945:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574346.880948:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574346.880951:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574346.880981:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574346.880985:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574346.880989:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574347.585103:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574347.585119:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574347.585124:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574347.904992:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574347.905001:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574347.905009:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574347.905014:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574347.905019:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574347.905023:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574347.905039:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574347.905044:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574347.905067:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574348.928936:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574348.928943:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574348.928948:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574348.928956:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574348.928963:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574348.928968:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574348.928982:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574348.928984:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574348.928987:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574349.633132:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574349.633135:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574349.633138:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574349.952914:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574349.952923:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574349.952927:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574349.952933:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574349.952939:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574349.952943:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574349.952948:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574349.952952:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574349.952958:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574349.953003:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574349.953006:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574349.953010:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574349.953014:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574349.953018:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574349.953021:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574349.953023:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574349.953024:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574349.953026:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574350.208926:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309710061, at 4309710080 00000100:00000010:0.0:1788574350.208943:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abf60. 00000020:00000001:0.0:1788574350.657042:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574350.657046:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574350.657048:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574350.976893:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574350.976917:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574350.976921:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574350.976927:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574350.976932:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574350.976937:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574350.976948:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574350.976951:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574350.976954:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574352.000913:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574352.000922:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574352.000926:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574352.000932:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574352.000938:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574352.000944:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574352.000957:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574352.000962:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574352.000967:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574352.001021:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574352.001026:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574352.001028:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574352.001031:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574352.001035:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574352.001037:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574352.001040:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574352.001042:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574352.001044:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574352.705038:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574352.705042:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574352.705044:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574353.024910:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574353.024917:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574353.024922:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574353.024927:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574353.024933:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574353.024938:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574353.024941:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574353.024946:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574353.024953:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574353.024989:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574353.024992:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574353.024994:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574353.024996:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574353.024998:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574353.025000:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574353.025002:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574353.025004:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574353.025007:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574354.048934:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574354.048939:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574354.048942:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574354.048946:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574354.048949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574354.048952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574354.048975:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574354.048979:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574354.048983:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574354.753053:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574354.753056:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574354.753059:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574355.073049:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574355.073053:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574355.073056:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574355.073060:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574355.073064:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574355.073069:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574355.073073:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574355.073074:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574355.073077:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574355.328913:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309715181, at 4309715200 00000100:00000010:0.0:1788574355.328932:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abea0. 00000020:00000001:0.0:1788574355.777023:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574355.777026:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574355.777029:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574356.096913:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574356.096922:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574356.096943:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574356.096948:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574356.096956:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574356.096962:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574356.096979:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574356.096984:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574356.096989:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574356.097020:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574356.097024:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574356.097027:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574356.097030:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574356.097038:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574356.097074:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574356.097076:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574356.097078:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574356.097080:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.120914:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574357.120919:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574357.120924:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574357.120934:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574357.120948:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574357.120952:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574357.120959:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574357.120965:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574357.120968:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574357.120983:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574357.120992:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574357.120996:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574357.121006:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574357.121012:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574357.121015:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574357.121017:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574357.121018:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574357.121021:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574357.825104:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574357.825108:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574357.825111:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574358.144990:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574358.145017:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574358.145023:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574358.145030:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574358.145039:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574358.145044:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574358.145095:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574358.145099:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574358.145103:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574359.168922:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574359.168931:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574359.168934:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574359.168942:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574359.168950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574359.168956:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574359.168971:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574359.168974:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574359.168977:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574359.872987:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574359.872992:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574359.872997:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574360.192932:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574360.192939:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574360.192944:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574360.192951:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574360.192959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574360.192964:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574360.192977:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574360.192982:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574360.192987:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574360.448922:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309720301, at 4309720320 00000100:00000010:0.0:1788574360.448938:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9aba80. 00000020:00000001:0.0:1788574360.897009:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574360.897013:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574360.897017:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574361.216972:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574361.216988:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574361.216998:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574361.217035:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574361.217045:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574361.217053:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574361.217082:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574361.217117:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574361.217123:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574362.241098:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574362.241113:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574362.241115:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574362.241117:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574362.241120:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574362.241122:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574362.241124:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574362.241126:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574362.241128:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574362.944965:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574362.944969:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574362.944973:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574363.264934:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574363.264944:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574363.264948:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574363.264954:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574363.264959:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574363.264965:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574363.264971:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574363.264977:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574363.264984:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574364.288891:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574364.288895:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574364.288898:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574364.288904:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574364.288911:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574364.288915:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574364.288927:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574364.288931:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574364.288935:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574364.992991:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574364.992996:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574364.993015:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574365.568915:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309725422, at 4309725440 00000100:00000010:0.0:1788574365.568935:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abd80. 00000020:00000001:0.0:1788574366.017039:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574366.017044:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574366.017047:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574366.336995:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574366.337000:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574366.337002:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574366.337004:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574366.337007:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574366.337009:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574366.337011:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574366.337012:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574366.337014:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574366.337053:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574366.337055:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574366.337057:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574366.337059:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574366.337061:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574366.337064:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574366.337065:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574366.337067:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574366.337069:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574367.360940:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574367.360948:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574367.360951:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574367.360955:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574367.360961:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574367.360964:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574367.360974:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574367.360977:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574367.360980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574368.064975:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574368.064978:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574368.064981:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574368.384916:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574368.384938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574368.384940:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574368.384947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574368.384952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574368.384956:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574368.384963:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574368.384968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574368.384973:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574369.408903:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574369.408913:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574369.408920:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574369.408930:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574369.408938:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574369.408942:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574369.408956:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574369.408960:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574369.408964:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574369.409012:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574369.409017:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574369.409022:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574369.409028:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574369.409036:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574369.409039:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574369.409042:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574369.409044:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574369.409046:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788574369.921222:0:2389:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788574369.921285:0:2389:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788574369.921298:0:2389:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574369.921303:0:2389:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788574369.921309:0:2389:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574369.921315:0:2389:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788574369.921321:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574369.921326:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc488504800. 00000400:00000200:0.0:1788574369.921333:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574369.921338:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574369.921369:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed000 00000400:00000010:0.0:1788574369.921373:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed000. 00000100:00000001:0.0:1788574369.921381:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574369.921392:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f5500 x1875440516688512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574391 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574369.921425:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574369.921430:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5500 x1875440516688512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574391 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574369.921439:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574369.921441:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574369.921454:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574369.921458:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc488504200. 00000400:00000200:0.0:1788574369.921462:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574369.921466:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574369.921470:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76edbb0 00000400:00000010:0.0:1788574369.921486:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76edbb0. 00000100:00000001:0.0:1788574369.921492:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574369.921502:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f6d80 x1875440516688640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574391 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574369.921529:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574369.921535:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6d80 x1875440516688640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574391 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574369.921544:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574369.921549:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574369.921555:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574369.921558:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc488504100. 00000400:00000200:0.0:1788574369.921561:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574369.921567:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574369.921574:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76edcc0 00000400:00000010:0.0:1788574369.921577:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76edcc0. 00000100:00000001:0.0:1788574369.921583:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574369.921589:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f5c00 x1875440516688768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574391 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574369.921612:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574369.921621:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5c00 x1875440516688768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574391 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574369.921638:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574369.921643:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574369.921656:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574369.921662:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc488504500. 00000400:00000200:0.0:1788574369.921683:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574369.921691:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574369.921696:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed660 00000400:00000010:0.0:1788574369.921700:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed660. 00000100:00000001:0.0:1788574369.921706:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574369.921716:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f6300 x1875440516688896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574391 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574369.921732:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574369.921737:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6300 x1875440516688896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574391 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574369.921753:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574369.921757:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574369.921764:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574369.921769:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc488504a00. 00000400:00000200:0.0:1788574369.921775:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574369.921783:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574369.921788:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed088 00000400:00000010:0.0:1788574369.921791:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed088. 00000100:00000001:0.0:1788574369.921797:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574369.921805:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac9f4380 x1875440516689024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574391 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574369.921872:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574369.921881:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4380 x1875440516689024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574391 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574369.921899:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574369.921901:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788574370.113006:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574370.113010:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574370.113014:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574370.432955:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574370.432962:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574370.432965:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574370.432970:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574370.432976:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574370.432981:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574370.432985:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574370.432988:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574370.432991:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574370.433033:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574370.433039:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574370.433044:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574370.433056:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574370.433070:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574370.433076:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574370.433094:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574370.433097:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574370.433102:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574370.688918:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309730541, at 4309730560 00000100:00000010:0.0:1788574370.688932:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab9c0. 00000020:00000001:0.0:1788574371.137035:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574371.137038:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574371.137041:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574371.456949:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574371.456954:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574371.456960:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574371.456966:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574371.456977:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574371.456988:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574371.457003:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574371.457021:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574371.457024:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574372.480936:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574372.480941:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574372.480945:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574372.480949:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574372.480956:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574372.480968:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574372.480993:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574372.480997:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574372.481001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574373.185060:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574373.185063:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574373.185066:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574373.504949:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574373.504955:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574373.504958:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574373.504963:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574373.504969:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574373.504974:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574373.504999:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574373.505002:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574373.505006:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574373.505032:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574373.505036:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574373.505039:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574373.505041:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574373.505044:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574373.505046:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574373.505048:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574373.505050:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574373.505060:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574374.528921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574374.528927:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574374.528932:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574374.528937:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574374.528942:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574374.528947:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574374.528977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574374.528981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574374.528985:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574375.232992:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574375.232996:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574375.233001:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574375.552933:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574375.552942:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574375.552947:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574375.552955:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574375.552964:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574375.552970:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574375.552976:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574375.552989:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574375.552996:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574375.808948:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309735662, at 4309735680 00000100:00000010:0.0:1788574375.808971:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab5a0. 00000020:00000001:0.0:1788574376.257065:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574376.257069:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574376.257072:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574376.576900:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574376.576908:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574376.576911:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574376.576918:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574376.576924:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574376.576930:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574376.576937:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574376.576943:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574376.576948:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574377.088919:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574377.088931:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574377.088934:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574377.088939:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574377.088944:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574377.088952:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15064 00000400:00000200:0.0:1788574377.088961:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574377.088986:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574377.088989:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574377.088994:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.088998:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574377.089000:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089003:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574377.089007:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574377.089009:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574377.089012:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574377.089015:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574377.089018:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574377.089020:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574377.089022:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089024:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574377.089025:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089026:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574377.089028:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574377.089031:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089032:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574377.089034:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574377.089037:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574377.089039:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.089041:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574377.089042:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574377.089047:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574377.089050:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574377.089053:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574377.089054:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.089056:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574377.089059:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574377.089062:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574377.089068:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574377.089077:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574377.089085:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516689792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574377.089096:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574377.089100:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574377.089105:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516689792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574377.089128:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574377.089132:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574377.089135:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574377.089137:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574377.089138:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574377.089141:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574377.089144:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15064 00000400:00000200:0.0:1788574377.089148:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574377.089152:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574377.089156:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574377.089161:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089164:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574377.089168:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089173:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574377.089177:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574377.089179:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574377.089182:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574377.089185:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574377.089188:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574377.089189:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574377.089191:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089192:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574377.089214:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089219:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574377.089226:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574377.089234:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089239:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574377.089245:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574377.089273:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574377.089278:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.089281:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574377.089285:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574377.089290:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.089295:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574377.089301:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574377.089305:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574377.089310:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574377.089312:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.089315:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574377.089319:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574377.089324:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574377.089329:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574377.089335:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574377.089344:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516689920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574377.089357:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574377.089361:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574377.089370:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516689920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574377.089384:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574377.089389:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574377.089394:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574377.089398:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574377.089405:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574377.089408:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574377.089413:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15064 00000400:00000200:0.0:1788574377.089428:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574377.089432:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574377.089434:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574377.089437:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089439:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574377.089441:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089442:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574377.089444:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574377.089445:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574377.089446:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574377.089447:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574377.089449:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574377.089450:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574377.089454:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089458:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574377.089462:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089467:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574377.089470:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574377.089476:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089481:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574377.089487:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574377.089492:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574377.089496:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.089501:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574377.089512:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574377.089519:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.089521:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574377.089534:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574377.089537:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574377.089543:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574377.089546:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.089550:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574377.089552:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574377.089556:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574377.089564:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574377.089570:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574377.089581:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516690048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574377.089598:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574377.089603:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574377.089610:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516690048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574377.089628:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574377.089633:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574377.089639:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574377.089643:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574377.089646:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574377.089653:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574377.089659:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15064 00000400:00000200:0.0:1788574377.089666:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574377.089671:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574377.089674:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574377.089680:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089685:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574377.089690:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089692:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574377.089694:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574377.089701:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574377.089705:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574377.089708:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574377.089712:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574377.089715:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574377.089725:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089730:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574377.089734:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089740:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574377.089748:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574377.089756:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089761:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574377.089767:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574377.089772:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574377.089776:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.089780:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574377.089784:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574377.089792:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.089798:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574377.089841:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574377.089847:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574377.089855:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574377.089859:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.089864:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574377.089867:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574377.089870:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574377.089878:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574377.089885:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574377.089894:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516690176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574377.089901:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574377.089904:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574377.089916:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516690176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574377.089932:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574377.089937:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574377.089944:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574377.089949:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574377.089953:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574377.089958:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574377.089965:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15064 00000400:00000200:0.0:1788574377.089970:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574377.089974:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574377.089977:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574377.089984:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.089989:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574377.089995:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.090000:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574377.090005:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574377.090008:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574377.090012:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574377.090015:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574377.090019:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574377.090023:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574377.090030:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.090034:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574377.090039:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574377.090042:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574377.090046:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574377.090051:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574377.090054:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574377.090062:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574377.090067:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574377.090073:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.090077:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574377.090080:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574377.090086:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.090092:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574377.090099:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574377.090107:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574377.090112:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574377.090115:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574377.090119:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574377.090123:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574377.090128:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574377.090136:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574377.090143:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574377.090152:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516690304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574377.090167:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574377.090172:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574377.090179:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516690304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574377.090200:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574377.090209:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15074) 00000100:00000001:0.0:1788574377.601042:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574377.601046:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574377.601047:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574377.601051:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574377.601054:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574377.601057:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574377.601059:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574377.601060:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574377.601062:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574378.305016:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574378.305025:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574378.305028:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574378.624934:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574378.624940:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574378.624944:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574378.624949:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574378.624954:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574378.624959:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574378.624965:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574378.624970:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574378.624976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574379.649074:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574379.649079:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574379.649082:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574379.649085:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574379.649088:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574379.649090:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574379.649093:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574379.649095:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574379.649097:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574380.160908:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788574380.160917:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788574380.160922:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788574380.161074:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574380.161088:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574380.161093:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574380.161096:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788574380.352982:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574380.352987:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574380.352991:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574380.672927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574380.672932:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574380.672935:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574380.672940:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574380.672946:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574380.672951:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574380.672960:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574380.672978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574380.672981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574380.928892:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309740781, at 4309740800 00000100:00000010:0.0:1788574380.928907:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab780. 00000020:00000001:0.0:1788574381.376949:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574381.376952:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574381.376955:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574381.696923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574381.696931:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574381.696936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574381.696944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574381.696952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574381.696958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574381.696973:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574381.696977:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574381.696981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.209162:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574382.209168:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574382.209173:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574382.209184:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516690432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.209203:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574382.209206:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574382.209211:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574382.209215:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574382.209220:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574382.209224:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516690432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574382.209231:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574382.209235:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574382.209239:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574382.209242:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574382.209248:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574382.209251:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574382.209258:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574382.209265:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574382.209280:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574382.209288:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516690432, portal 25 00000100:00000001:0.0:1788574382.209292:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574382.209294:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788574382.209315:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516690432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.209330:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574382.209335:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574382.209339:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574382.209344:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516690432, offset 0 00000400:00000200:0.0:1788574382.209354:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574382.209364:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574382.209373:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574382.209387:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574382.209393:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574382.209400:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574382.209406:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574382.209413:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547046:547046:256:4294967295] 192.168.202.128@tcp LPNI seq info [546969:546969:8:4294967295] 00000400:00000200:0.0:1788574382.209420:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574382.209434:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574382.209444:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574382.209451:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574382.209460:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574382.209465:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574382.209486:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574382.209488:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574382.209495:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574382.209502:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574382.209509:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574382.209512:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574382.209517:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574382.209529:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516690432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.209543:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574382.209548:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516690432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.209556:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.209559:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574382.209563:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574382.209568:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574382.209571:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309745901 00000100:00000010:0.0:1788574382.209574:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788574382.209582:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574382.209588:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309745901 00000100:00000010:0.0:1788574382.209593:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50120. 00000100:00000001:0.0:1788574382.209598:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574382.209602:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574382.209605:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.209610:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574382.209615:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574382/real 1788574382] req@ffff8fc4ac822a00 x1875440516690432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574382.209626:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574382.209628:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574382.209631:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574382.209634:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574382.209637:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574382.209641:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574382.209647:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516690432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574382.209661:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516690432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.209669:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574382.209678:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.209681:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574382.209684:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574382.209692:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516690432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.209705:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574382.209714:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516690432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.209726:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574382.209731:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.209735:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574382.209737:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.209740:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574382.209744:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574382.209749:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574382.209756:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574382.209760:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574382.209767:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516690432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574382.209781:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516690432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574382.209787:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.209791:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574382.209796:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516690432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.209805:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574382.209824:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574382.209827:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574382.209829:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574382.209831:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574382.209836:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574382.209838:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574382.209840:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788574382.209843:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574382.209845:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574382.209849:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788574382.209851:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574382.209853:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.209855:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.209861:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574382.209863:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.209867:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574382.209870:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574382.209872:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574382.209878:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516690560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.209887:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574382.209889:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574382.209893:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574382.209895:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574382.209899:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574382.209903:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516690560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574382.209908:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574382.209911:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574382.209913:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574382.209917:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574382.209921:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574382.209924:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574382.209930:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574382.209933:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574382.209939:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574382.209944:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516690560, portal 10 00000100:00000001:0.0:1788574382.209947:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574382.209951:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788574382.209961:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516690560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.209975:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574382.209981:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574382.209985:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574382.209988:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516690560, offset 0 00000400:00000200:0.0:1788574382.209994:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574382.209999:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574382.210018:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574382.210022:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574382.210025:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574382.210028:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574382.210030:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574382.210035:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547047:547047:256:4294967295] 192.168.202.128@tcp LPNI seq info [546970:546970:8:4294967295] 00000400:00000200:0.0:1788574382.210044:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574382.210058:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574382.210064:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574382.210070:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574382.210080:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574382.210085:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574382.210101:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574382.210105:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574382.210110:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574382.210116:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574382.210121:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574382.210123:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574382.210128:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574382.210137:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516690560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210157:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574382.210165:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516690560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210220:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.210222:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574382.210227:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574382.210234:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574382.210239:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574382.210242:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309742099 00000100:00000001:0.0:1788574382.210244:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309745901 00000100:00000001:0.0:1788574382.210247:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309742099 00000100:00000001:0.0:1788574382.210248:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309745901 00000100:00000001:0.0:1788574382.210250:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574382.210252:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.210254:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574382.210257:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574382/real 1788574382] req@ffff8fc4ac823100 x1875440516690560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574382.210263:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574382.210264:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574382.210265:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574382.210267:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574382.210268:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574382.210270:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574382.210272:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516690560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574382.210278:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516690560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210282:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574382.210283:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210284:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574382.210285:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574382.210287:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516690560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210298:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574382.210301:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516690560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210318:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574382.210319:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210320:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574382.210322:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210323:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574382.210324:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574382.210326:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574382.210329:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574382.210330:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574382.210333:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516690560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574382.210339:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516690560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574382.210342:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574382.210344:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516690688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210348:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574382.210349:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574382.210350:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574382.210351:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574382.210353:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574382.210355:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516690688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574382.210356:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574382.210358:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574382.210363:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574382.210364:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574382.210366:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574382.210368:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574382.210370:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574382.210372:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574382.210375:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574382.210377:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516690688, portal 10 00000100:00000001:0.0:1788574382.210378:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574382.210379:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788574382.210384:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516690688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210388:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574382.210390:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574382.210391:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574382.210393:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516690688, offset 0 00000400:00000200:0.0:1788574382.210395:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574382.210398:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574382.210400:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574382.210402:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574382.210404:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574382.210406:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574382.210407:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574382.210410:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547048:547048:256:4294967295] 192.168.202.128@tcp LPNI seq info [546971:546971:8:4294967295] 00000400:00000200:0.0:1788574382.210412:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574382.210417:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574382.210421:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574382.210423:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574382.210426:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574382.210428:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574382.210434:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574382.210439:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574382.210441:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574382.210444:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574382.210446:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574382.210447:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574382.210449:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574382.210452:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516690688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210457:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574382.210459:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516690688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210463:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.210464:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574382.210466:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574382.210469:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574382.210471:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574382.210472:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309742106 00000100:00000001:0.0:1788574382.210474:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309745901 00000100:00000001:0.0:1788574382.210475:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309742106 00000100:00000001:0.0:1788574382.210477:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309745901 00000100:00000001:0.0:1788574382.210478:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574382.210479:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.210481:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574382.210483:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574382/real 1788574382] req@ffff8fc4ac821500 x1875440516690688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574382.210488:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574382.210489:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574382.210490:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574382.210491:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574382.210493:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574382.210497:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574382.210500:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516690688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574382.210505:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516690688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210509:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574382.210510:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210512:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574382.210513:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574382.210515:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516690688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210519:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574382.210522:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516690688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210526:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574382.210527:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210528:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574382.210529:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210530:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574382.210532:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574382.210533:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574382.210535:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574382.210536:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574382.210539:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516690688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574382.210544:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516690688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574382.210546:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210550:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574382.210553:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516690560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210557:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574382.210559:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574382.210560:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574382.210561:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574382.210562:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574382.210563:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574382.210565:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574382.210566:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788574382.210568:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574382.210568:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574382.210569:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788574382.210571:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574382.210572:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210573:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574382.210576:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516690688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210579:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574382.210580:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574382.210581:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574382.210582:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574382.210583:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574382.210583:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574382.210585:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574382.210585:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788574382.210586:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574382.210587:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574382.210588:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788574382.210589:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574382.210590:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210591:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210594:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574382.210595:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.210596:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574382.210600:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574382.210601:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574382.210604:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516690816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210608:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574382.210609:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574382.210610:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574382.210612:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574382.210613:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574382.210615:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516690816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574382.210617:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574382.210618:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574382.210619:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574382.210620:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574382.210622:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574382.210623:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574382.210625:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574382.210627:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574382.210629:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574382.210631:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516690816, portal 4 00000100:00000001:0.0:1788574382.210633:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574382.210633:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788574382.210637:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516690816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210642:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574382.210643:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574382.210644:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574382.210646:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516690816, offset 0 00000400:00000200:0.0:1788574382.210648:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574382.210651:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574382.210653:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574382.210655:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574382.210657:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574382.210662:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574382.210664:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574382.210666:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547049:547049:256:4294967295] 192.168.202.128@tcp LPNI seq info [546972:546972:8:4294967295] 00000400:00000200:0.0:1788574382.210669:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574382.210673:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574382.210676:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574382.210678:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574382.210681:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574382.210682:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574382.210689:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574382.210691:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574382.210692:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574382.210695:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574382.210697:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574382.210698:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574382.210699:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574382.210702:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516690816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210707:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574382.210709:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516690816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210713:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.210714:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574382.210716:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574382.210718:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574382.210720:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574382.210721:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309742246 00000100:00000001:0.0:1788574382.210723:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309745901 00000100:00000001:0.0:1788574382.210725:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309742246 00000100:00000001:0.0:1788574382.210731:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309745901 00000100:00000001:0.0:1788574382.210733:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574382.210734:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.210735:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574382.210738:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574382/real 1788574382] req@ffff8fc4ac823b80 x1875440516690816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574382.210743:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574382.210744:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574382.210745:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574382.210747:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574382.210748:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574382.210749:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574382.210751:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516690816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574382.210757:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516690816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210761:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574382.210762:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210763:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574382.210764:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574382.210766:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516690816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210770:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574382.210772:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516690816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210776:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574382.210778:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210779:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574382.210780:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210783:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574382.210785:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574382.210787:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574382.210789:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574382.210790:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574382.210793:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516690816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574382.210798:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516690816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574382.210800:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574382.210802:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516690944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210806:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574382.210822:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574382.210824:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574382.210825:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574382.210826:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574382.210828:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516690944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574382.210830:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574382.210831:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574382.210832:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574382.210833:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574382.210836:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574382.210837:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574382.210839:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574382.210841:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574382.210843:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574382.210845:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516690944, portal 4 00000100:00000001:0.0:1788574382.210847:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574382.210848:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788574382.210851:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516690944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210859:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574382.210861:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574382.210862:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574382.210864:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516690944, offset 0 00000400:00000200:0.0:1788574382.210866:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574382.210869:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574382.210872:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574382.210874:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574382.210875:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574382.210877:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574382.210878:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574382.210881:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547050:547050:256:4294967295] 192.168.202.128@tcp LPNI seq info [546973:546973:8:4294967295] 00000400:00000200:0.0:1788574382.210883:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574382.210888:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574382.210891:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574382.210892:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574382.210895:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574382.210897:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574382.210904:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574382.210905:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574382.210907:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574382.210909:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574382.210911:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574382.210912:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574382.210914:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574382.210916:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516690944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210922:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574382.210924:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516690944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574407 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210935:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.210936:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574382.210939:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574382.210941:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574382.210943:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574382.210945:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309742249 00000100:00000001:0.0:1788574382.210946:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309745901 00000100:00000001:0.0:1788574382.210948:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309742249 00000100:00000001:0.0:1788574382.210949:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309745901 00000100:00000001:0.0:1788574382.210951:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574382.210952:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.210953:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574382.210955:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574382/real 1788574382] req@ffff8fc4ac820000 x1875440516690944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574382.210960:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574382.210961:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574382.210962:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574382.210964:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574382.210965:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574382.210966:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574382.210968:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516690944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574382.210973:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516690944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210977:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574382.210978:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210979:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574382.210980:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574382.210982:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516690944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210990:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574382.210993:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516690944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.210997:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574382.210998:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.210999:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574382.211000:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.211001:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574382.211002:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574382.211004:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574382.211006:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574382.211008:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574382.211010:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516690944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574382.211015:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516690944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574382.211017:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.211018:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574382.211020:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516690816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.211024:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574382.211025:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574382.211026:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574382.211027:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574382.211028:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574382.211029:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574382.211030:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574382.211032:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788574382.211033:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574382.211037:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574382.211038:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788574382.211040:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574382.211041:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.211042:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574382.211044:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516690944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574407 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574382.211048:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574382.211049:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574382.211050:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574382.211051:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574382.211052:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574382.211053:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574382.211054:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574382.211055:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788574382.211056:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574382.211057:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574382.211057:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788574382.211059:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574382.211059:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.211061:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.211063:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574382.211064:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.211065:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574382.211066:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574382.720944:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574382.720949:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574382.720953:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574382.720957:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574382.720968:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574382.720973:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.720983:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574382.720986:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574382.720989:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574383.425026:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574383.425030:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574383.425033:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574383.745079:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574383.745081:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574383.745083:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574383.745085:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574383.745088:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574383.745090:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574383.745091:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574383.745092:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574383.745094:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574384.768918:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574384.768924:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574384.768929:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574384.768935:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574384.768942:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574384.768948:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574384.768955:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574384.768961:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574384.768967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574384.768984:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574384.768988:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574384.768991:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574384.768994:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574384.768996:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574384.768998:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574384.769000:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574384.769001:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574384.769002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574385.473012:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574385.473015:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574385.473018:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574385.792935:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574385.792942:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574385.792948:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574385.792952:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574385.792957:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574385.792962:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574385.792977:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574385.792982:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574385.793011:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574385.793036:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574385.793042:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574385.793047:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574385.793055:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574385.793060:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574385.793064:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574385.793066:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574385.793069:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574385.793072:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574386.048915:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309745901, at 4309745920 00000100:00000010:0.0:1788574386.048936:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788574386.496983:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574386.496987:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574386.496990:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574386.816918:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574386.816926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574386.816930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574386.816937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574386.816945:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574386.816949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574386.816966:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574386.816972:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574386.816978:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.329110:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574387.329121:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574387.329127:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574387.329135:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516691072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.329148:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574387.329152:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574387.329159:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574387.329163:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574387.329166:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574387.329174:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516691072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574387.329191:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574387.329197:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574387.329201:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574387.329206:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574387.329211:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574387.329214:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574387.329223:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574387.329228:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574387.329235:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574387.329243:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516691072, portal 25 00000100:00000001:0.0:1788574387.329248:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574387.329251:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788574387.329258:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516691072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.329268:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574387.329273:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574387.329277:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574387.329281:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516691072, offset 0 00000400:00000200:0.0:1788574387.329292:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574387.329300:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574387.329310:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574387.329320:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574387.329328:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574387.329336:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574387.329345:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574387.329355:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547051:547051:256:4294967295] 192.168.202.128@tcp LPNI seq info [546974:546974:8:4294967295] 00000400:00000200:0.0:1788574387.329384:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574387.329401:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574387.329413:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574387.329420:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574387.329431:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574387.329441:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574387.329460:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574387.329462:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574387.329471:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574387.329484:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574387.329493:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574387.329499:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574387.329509:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574387.329519:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516691072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.329539:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574387.329548:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516691072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.329562:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.329567:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574387.329581:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574387.329592:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574387.329598:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309751021 00000100:00000010:0.0:1788574387.329608:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788574387.329617:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574387.329626:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309751021 00000100:00000010:0.0:1788574387.329635:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50900. 00000100:00000001:0.0:1788574387.329642:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574387.329647:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574387.329651:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.329656:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574387.329665:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574387/real 1788574387] req@ffff8fc4ac820000 x1875440516691072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574387.329685:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574387.329687:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574387.329696:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574387.329700:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574387.329705:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574387.329711:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574387.329716:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516691072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574387.329729:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516691072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.329740:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574387.329743:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.329748:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574387.329751:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574387.329759:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516691072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.329763:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574387.329769:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516691072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.329788:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574387.329796:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.329802:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574387.329828:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.329833:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574387.329839:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574387.329847:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574387.329858:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574387.329865:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574387.329874:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516691072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574387.329889:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516691072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574387.329907:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.329912:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574387.329920:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516691072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.329939:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574387.329942:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574387.329946:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574387.329953:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574387.329959:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574387.329964:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574387.329969:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574387.329975:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788574387.329982:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574387.329986:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574387.329990:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788574387.329994:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574387.329997:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330001:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330007:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574387.330010:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.330013:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574387.330017:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574387.330018:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574387.330026:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516691200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330046:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574387.330049:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574387.330056:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574387.330061:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574387.330066:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574387.330072:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516691200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574387.330077:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574387.330079:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574387.330080:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574387.330088:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574387.330091:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574387.330093:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574387.330097:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574387.330099:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574387.330101:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574387.330104:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516691200, portal 10 00000100:00000001:0.0:1788574387.330105:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574387.330106:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788574387.330112:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516691200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330116:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574387.330118:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574387.330119:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574387.330121:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516691200, offset 0 00000400:00000200:0.0:1788574387.330123:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574387.330126:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574387.330129:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574387.330131:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574387.330133:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574387.330135:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574387.330136:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574387.330138:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547052:547052:256:4294967295] 192.168.202.128@tcp LPNI seq info [546975:546975:8:4294967295] 00000400:00000200:0.0:1788574387.330141:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574387.330146:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574387.330149:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574387.330151:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574387.330154:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574387.330156:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574387.330164:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574387.330166:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574387.330173:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574387.330176:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574387.330178:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574387.330179:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574387.330181:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574387.330184:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516691200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330189:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574387.330191:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516691200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330196:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.330197:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574387.330199:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574387.330201:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574387.330204:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574387.330205:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309747219 00000100:00000001:0.0:1788574387.330207:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309751021 00000100:00000001:0.0:1788574387.330208:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309747219 00000100:00000001:0.0:1788574387.330210:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309751021 00000100:00000001:0.0:1788574387.330212:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574387.330213:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.330214:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574387.330217:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574387/real 1788574387] req@ffff8fc4ac823b80 x1875440516691200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574387.330221:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574387.330222:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574387.330223:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574387.330225:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574387.330226:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574387.330227:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574387.330233:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516691200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574387.330239:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516691200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330243:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574387.330244:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330245:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574387.330246:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574387.330248:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516691200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330253:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574387.330255:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516691200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330260:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574387.330261:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330262:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574387.330263:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330264:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574387.330266:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574387.330267:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574387.330269:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574387.330271:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574387.330273:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516691200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574387.330278:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516691200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574387.330281:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574387.330283:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516691328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330294:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574387.330295:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574387.330297:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574387.330298:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574387.330299:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574387.330301:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516691328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574387.330303:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574387.330305:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574387.330306:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574387.330307:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574387.330309:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574387.330311:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574387.330314:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574387.330317:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574387.330320:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574387.330323:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516691328, portal 10 00000100:00000001:0.0:1788574387.330325:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574387.330327:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788574387.330332:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516691328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330339:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574387.330342:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574387.330344:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574387.330347:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516691328, offset 0 00000400:00000200:0.0:1788574387.330351:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574387.330370:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574387.330375:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574387.330378:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574387.330382:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574387.330384:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574387.330385:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574387.330392:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547053:547053:256:4294967295] 192.168.202.128@tcp LPNI seq info [546976:546976:8:4294967295] 00000400:00000200:0.0:1788574387.330396:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574387.330401:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574387.330404:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574387.330406:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574387.330409:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574387.330411:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574387.330419:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574387.330420:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574387.330422:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574387.330425:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574387.330427:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574387.330428:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574387.330430:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574387.330433:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516691328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330439:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574387.330441:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516691328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330445:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.330446:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574387.330448:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574387.330451:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574387.330453:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574387.330454:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309747226 00000100:00000001:0.0:1788574387.330456:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309751021 00000100:00000001:0.0:1788574387.330457:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309747226 00000100:00000001:0.0:1788574387.330459:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309751021 00000100:00000001:0.0:1788574387.330460:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574387.330465:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.330466:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574387.330469:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574387/real 1788574387] req@ffff8fc4ac821500 x1875440516691328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574387.330475:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574387.330476:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574387.330478:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574387.330479:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574387.330480:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574387.330482:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574387.330484:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516691328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574387.330489:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516691328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330493:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574387.330494:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330495:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574387.330496:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574387.330498:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516691328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330502:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574387.330504:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516691328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330509:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574387.330511:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330512:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574387.330513:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330515:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574387.330518:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574387.330521:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574387.330529:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574387.330532:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574387.330536:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516691328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574387.330545:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516691328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574387.330549:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574387.330553:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516691456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330560:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574387.330562:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574387.330565:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574387.330566:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574387.330569:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574387.330571:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516691456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574387.330573:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574387.330574:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574387.330576:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574387.330577:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574387.330580:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574387.330581:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574387.330584:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574387.330586:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574387.330588:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574387.330590:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516691456, portal 4 00000100:00000001:0.0:1788574387.330592:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574387.330593:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788574387.330597:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516691456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330601:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574387.330607:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574387.330608:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574387.330610:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516691456, offset 0 00000400:00000200:0.0:1788574387.330612:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574387.330616:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574387.330618:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574387.330620:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574387.330622:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574387.330624:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574387.330625:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574387.330627:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547054:547054:256:4294967295] 192.168.202.128@tcp LPNI seq info [546977:546977:8:4294967295] 00000400:00000200:0.0:1788574387.330630:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574387.330634:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574387.330637:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574387.330639:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574387.330642:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574387.330644:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574387.330651:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574387.330652:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574387.330654:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574387.330657:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574387.330659:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574387.330660:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574387.330662:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574387.330664:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516691456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330670:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574387.330672:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516691456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330676:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.330677:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574387.330684:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574387.330686:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574387.330689:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574387.330690:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309747366 00000100:00000001:0.0:1788574387.330692:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309751021 00000100:00000001:0.0:1788574387.330693:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309747366 00000100:00000001:0.0:1788574387.330695:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309751021 00000100:00000001:0.0:1788574387.330696:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574387.330697:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.330698:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574387.330701:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574387/real 1788574387] req@ffff8fc4ac823100 x1875440516691456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574387.330706:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574387.330707:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574387.330708:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574387.330710:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574387.330711:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574387.330712:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574387.330714:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516691456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574387.330720:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516691456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330724:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574387.330725:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330726:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574387.330727:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574387.330729:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516691456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330733:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574387.330735:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516691456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330743:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574387.330745:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330746:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574387.330747:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330748:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574387.330750:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574387.330751:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574387.330754:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574387.330757:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574387.330761:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516691456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574387.330769:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516691456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574387.330773:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330775:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574387.330778:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516691200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330785:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574387.330787:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574387.330789:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574387.330792:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574387.330793:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574387.330794:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574387.330796:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574387.330798:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788574387.330801:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574387.330803:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574387.330804:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788574387.330823:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574387.330824:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330830:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574387.330834:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516691328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330838:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574387.330839:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574387.330841:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574387.330842:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574387.330843:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574387.330844:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574387.330845:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574387.330846:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788574387.330848:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574387.330849:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574387.330850:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788574387.330851:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574387.330852:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330853:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574387.330855:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516691456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330859:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574387.330860:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574387.330861:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574387.330863:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574387.330864:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574387.330864:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574387.330865:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574387.330866:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788574387.330867:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574387.330868:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574387.330869:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788574387.330870:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574387.330871:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330872:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.330875:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574387.330875:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.330877:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574387.330880:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574387.330882:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574387.330884:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516691584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330889:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574387.330890:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574387.330891:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574387.330893:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574387.330894:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574387.330896:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516691584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574387.330897:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574387.330899:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574387.330900:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574387.330901:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574387.330903:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574387.330904:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574387.330907:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574387.330908:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574387.330911:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574387.330913:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516691584, portal 4 00000100:00000001:0.0:1788574387.330914:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574387.330915:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788574387.330919:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516691584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.330924:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574387.330925:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574387.330926:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574387.330928:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516691584, offset 0 00000400:00000200:0.0:1788574387.330930:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574387.330933:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574387.330935:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574387.330937:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574387.330939:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574387.330944:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574387.330946:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574387.330949:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547055:547055:256:4294967295] 192.168.202.128@tcp LPNI seq info [546978:546978:8:4294967295] 00000400:00000200:0.0:1788574387.330954:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574387.330961:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574387.330967:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574387.330971:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574387.330975:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574387.330978:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574387.330989:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574387.330991:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574387.330994:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574387.330996:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574387.330998:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574387.330999:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574387.331002:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574387.331005:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516691584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.331010:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574387.331012:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516691584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574412 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.331016:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.331017:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574387.331020:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574387.331022:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574387.331025:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574387.331027:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309747369 00000100:00000001:0.0:1788574387.331029:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309751021 00000100:00000001:0.0:1788574387.331032:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309747369 00000100:00000001:0.0:1788574387.331041:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309751021 00000100:00000001:0.0:1788574387.331043:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574387.331045:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.331047:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574387.331052:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574387/real 1788574387] req@ffff8fc4ac822a00 x1875440516691584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574387.331062:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574387.331063:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574387.331065:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574387.331068:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574387.331070:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574387.331073:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574387.331076:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516691584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574387.331083:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516691584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.331088:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574387.331089:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.331090:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574387.331091:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574387.331093:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516691584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.331098:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574387.331101:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516691584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.331105:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574387.331106:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.331107:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574387.331108:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.331113:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574387.331115:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574387.331117:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574387.331119:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574387.331121:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574387.331123:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516691584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574387.331128:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516691584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574387.331131:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.331132:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574387.331134:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516691584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574412 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574387.331137:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574387.331138:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574387.331140:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574387.331142:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574387.331143:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574387.331144:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574387.331145:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574387.331146:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788574387.331148:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574387.331149:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574387.331150:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788574387.331152:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574387.331153:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.331154:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.331157:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574387.331158:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.331159:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574387.331160:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574387.840910:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574387.840920:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574387.840941:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574387.840947:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574387.840955:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574387.840958:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.840978:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574387.840981:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574387.840986:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574388.545008:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574388.545012:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574388.545015:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574388.864910:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574388.864918:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574388.864923:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574388.864930:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574388.864937:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574388.864943:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574388.864969:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574388.864972:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574388.864977:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574389.888886:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574389.888893:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574389.888898:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574389.888904:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574389.888911:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574389.888917:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574389.888930:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574389.888934:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574389.888937:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574390.593061:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574390.593065:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574390.593068:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574390.912916:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574390.912921:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574390.912924:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574390.912932:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574390.912936:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574390.912943:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574390.912948:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574390.912964:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574390.912970:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574390.912987:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574390.912991:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574390.912995:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574390.912997:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574390.913004:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574390.913010:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574390.913015:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574390.913017:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574390.913020:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574391.168936:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309751021, at 4309751040 00000100:00000010:0.0:1788574391.168952:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788574391.616998:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574391.617001:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574391.617004:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574391.936900:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574391.936907:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574391.936910:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574391.936916:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574391.936922:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574391.936926:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574391.936930:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574391.936935:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574391.936940:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574392.448903:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574392.448910:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574392.448913:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574392.448916:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574392.448920:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574392.448927:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15079 00000400:00000200:0.0:1788574392.448935:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574392.448939:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574392.448942:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574392.448944:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.448963:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574392.448966:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.448969:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574392.448973:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574392.448975:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574392.448978:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574392.448981:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574392.448984:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574392.448987:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574392.448990:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.448993:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574392.448995:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.448997:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574392.449000:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574392.449004:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449007:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574392.449009:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574392.449012:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574392.449015:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449016:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574392.449018:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574392.449023:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574392.449026:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574392.449030:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574392.449031:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449033:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574392.449036:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574392.449040:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574392.449046:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574392.449051:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574392.449059:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516691712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574392.449075:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574392.449079:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574392.449083:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516691712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574392.449102:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574392.449105:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574392.449107:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574392.449108:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574392.449110:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574392.449111:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574392.449113:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15079 00000400:00000200:0.0:1788574392.449116:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574392.449119:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574392.449121:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574392.449128:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449130:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574392.449132:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449134:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574392.449135:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574392.449137:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574392.449141:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574392.449144:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574392.449148:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574392.449151:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574392.449155:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449157:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574392.449161:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449163:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574392.449171:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574392.449176:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449181:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574392.449184:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574392.449190:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574392.449194:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449197:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574392.449200:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574392.449203:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449204:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574392.449206:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574392.449208:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574392.449209:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574392.449210:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449212:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574392.449213:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574392.449214:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574392.449217:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574392.449220:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574392.449226:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516691840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574392.449236:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574392.449238:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574392.449245:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516691840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574392.449254:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574392.449258:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574392.449263:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574392.449267:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574392.449269:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574392.449270:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574392.449280:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15079 00000400:00000200:0.0:1788574392.449286:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574392.449291:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574392.449293:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574392.449298:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449300:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574392.449302:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449304:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574392.449309:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574392.449310:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574392.449311:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574392.449313:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574392.449315:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574392.449320:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574392.449325:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449330:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574392.449333:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449335:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574392.449339:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574392.449345:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449349:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574392.449352:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574392.449356:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574392.449360:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449365:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574392.449370:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574392.449376:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449379:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574392.449385:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574392.449390:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574392.449394:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574392.449436:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449439:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574392.449442:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574392.449446:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574392.449451:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574392.449457:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574392.449463:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516691968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574392.449477:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574392.449481:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574392.449486:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516691968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574392.449497:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574392.449500:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574392.449505:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574392.449506:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574392.449507:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574392.449512:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574392.449516:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15079 00000400:00000200:0.0:1788574392.449523:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574392.449528:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574392.449533:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574392.449539:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449550:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574392.449555:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449558:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574392.449560:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574392.449562:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574392.449565:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574392.449568:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574392.449571:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574392.449581:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574392.449590:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449594:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574392.449599:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449603:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574392.449610:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574392.449617:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449621:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574392.449626:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574392.449631:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574392.449637:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449643:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574392.449648:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574392.449653:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449656:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574392.449663:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574392.449669:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574392.449675:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574392.449679:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449684:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574392.449686:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574392.449689:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574392.449697:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574392.449703:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574392.449716:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516692096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574392.449734:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574392.449738:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574392.449746:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516692096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574392.449754:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574392.449763:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574392.449767:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574392.449770:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574392.449772:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574392.449775:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574392.449779:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15079 00000400:00000200:0.0:1788574392.449784:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574392.449790:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574392.449794:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574392.449800:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449804:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574392.449825:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449827:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574392.449829:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574392.449830:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574392.449832:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574392.449834:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574392.449838:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574392.449842:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574392.449849:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449852:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574392.449856:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449861:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574392.449865:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574392.449870:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574392.449872:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574392.449876:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574392.449879:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574392.449883:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449885:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574392.449894:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574392.449898:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449901:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574392.449907:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574392.449911:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574392.449914:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574392.449917:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.449919:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574392.449921:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574392.449925:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574392.449930:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574392.449936:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574392.449946:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516692224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574392.449957:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574392.449963:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574392.449972:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516692224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574392.449989:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574392.449997:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15090) 00000100:00000001:0.0:1788574392.960905:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574392.960913:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574392.960915:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574392.960921:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574392.960926:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574392.960931:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574392.960937:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574392.960941:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574392.960947:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574392.960974:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574392.960978:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574392.960981:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574392.960987:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574392.960990:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574392.960999:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574392.961001:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574392.961002:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574392.961004:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574393.664959:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574393.664964:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574393.664967:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574393.984936:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574393.984941:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574393.984944:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574393.984948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574393.984955:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574393.984961:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574393.984966:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574393.984970:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574393.984973:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574395.008912:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574395.008921:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574395.008926:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574395.008935:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574395.008942:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574395.008947:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574395.008950:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574395.008954:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574395.008961:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574395.713025:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574395.713030:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574395.713034:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574396.032891:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574396.032896:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574396.032898:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574396.032902:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574396.032906:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574396.032912:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574396.032934:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574396.032938:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574396.032943:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574396.032993:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574396.033004:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574396.033006:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574396.033010:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574396.033013:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574396.033018:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574396.033031:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574396.033043:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574396.033059:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574396.288927:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309756141, at 4309756160 00000100:00000010:0.0:1788574396.288948:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab8a0. 00000020:00000001:0.0:1788574396.736983:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574396.736986:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574396.736988:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574397.056900:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574397.056907:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574397.056912:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574397.056917:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574397.056921:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574397.056927:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574397.056945:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574397.056948:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574397.056956:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574397.569006:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574397.569020:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574397.569023:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574397.569028:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574397.569032:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574397.569041:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15085 00000400:00000200:0.0:1788574397.569051:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574397.569058:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574397.569061:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574397.569064:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569066:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574397.569068:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569093:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574397.569097:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574397.569099:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574397.569102:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574397.569105:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574397.569107:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574397.569109:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574397.569112:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569114:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574397.569115:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569116:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574397.569119:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574397.569121:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569123:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574397.569126:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574397.569128:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574397.569131:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.569133:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574397.569142:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574397.569151:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574397.569154:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574397.569158:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574397.569159:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.569162:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574397.569165:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574397.569170:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574397.569178:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574397.569186:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574397.569195:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516692352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574397.569204:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574397.569209:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574397.569218:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516692352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574397.569238:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574397.569241:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574397.569244:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574397.569245:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574397.569246:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574397.569248:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574397.569250:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15085 00000400:00000200:0.0:1788574397.569253:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574397.569255:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574397.569257:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574397.569259:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569261:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574397.569262:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569264:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574397.569266:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574397.569267:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574397.569269:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574397.569272:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574397.569274:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574397.569276:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574397.569280:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569282:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574397.569284:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569287:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574397.569290:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574397.569294:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569297:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574397.569305:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574397.569313:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574397.569317:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.569321:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574397.569330:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574397.569338:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.569343:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574397.569349:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574397.569356:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574397.569368:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574397.569373:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.569381:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574397.569386:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574397.569392:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574397.569406:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574397.569417:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574397.569433:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516692480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574397.569455:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574397.569458:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574397.569480:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516692480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574397.569505:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574397.569511:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574397.569520:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574397.569524:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574397.569536:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574397.569552:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574397.569561:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15085 00000400:00000200:0.0:1788574397.569576:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574397.569585:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574397.569590:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574397.569606:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569610:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574397.569612:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569621:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574397.569628:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574397.569631:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574397.569637:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574397.569646:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574397.569649:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574397.569653:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574397.569662:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569666:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574397.569670:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569674:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574397.569678:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574397.569683:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569687:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574397.569691:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574397.569695:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574397.569698:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.569702:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574397.569704:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574397.569708:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.569713:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574397.569721:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574397.569732:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574397.569740:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574397.569744:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.569752:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574397.569758:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574397.569763:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574397.569770:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574397.569797:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574397.569848:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516692608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574397.569858:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574397.569861:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574397.569866:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516692608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574397.569876:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574397.569881:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574397.569885:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574397.569887:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574397.569889:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574397.569893:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574397.569897:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15085 00000400:00000200:0.0:1788574397.569913:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574397.569922:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574397.569934:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574397.569944:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569956:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574397.569964:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.569971:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574397.569983:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574397.569988:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574397.569994:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574397.569998:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574397.570002:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574397.570004:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574397.570009:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.570011:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574397.570034:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.570041:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574397.570046:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574397.570059:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574397.570066:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574397.570072:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574397.570077:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574397.570082:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.570090:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574397.570094:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574397.570103:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.570110:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574397.570116:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574397.570123:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574397.570130:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574397.570134:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.570140:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574397.570144:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574397.570150:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574397.570158:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574397.570166:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574397.570192:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516692736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574397.570208:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574397.570214:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574397.570228:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516692736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574397.570248:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574397.570256:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574397.570262:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574397.570264:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574397.570276:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574397.570284:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574397.570294:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15085 00000400:00000200:0.0:1788574397.570304:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574397.570312:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574397.570320:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574397.570334:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.570343:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574397.570350:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.570356:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574397.570360:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574397.570364:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574397.570370:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574397.570388:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574397.570396:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574397.570402:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574397.570415:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.570423:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574397.570431:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574397.570437:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574397.570445:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574397.570450:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574397.570453:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574397.570460:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574397.570478:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574397.570486:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.570493:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574397.570500:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574397.570507:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.570511:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574397.570531:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574397.570537:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574397.570544:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574397.570548:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574397.570553:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574397.570557:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574397.570562:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574397.570573:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574397.570579:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574397.570592:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516692864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574397.570652:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574397.570658:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574397.570673:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516692864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574397.570700:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574397.570716:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15095) 00000100:00000001:0.0:1788574398.080909:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574398.080917:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574398.080920:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574398.080925:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574398.080931:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574398.080937:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574398.080945:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574398.080950:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574398.080955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574398.785005:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574398.785008:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574398.785011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574399.104909:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574399.104914:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574399.104919:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574399.104927:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574399.104934:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574399.104941:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574399.104964:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574399.104968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574399.104972:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574400.128927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574400.128933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574400.128937:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574400.128945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574400.128953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574400.128959:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574400.128965:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574400.128969:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574400.128973:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574400.832951:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574400.832955:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574400.832959:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574401.152902:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574401.152907:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574401.152910:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574401.152916:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574401.152922:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574401.152927:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574401.152942:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574401.152946:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574401.152950:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574401.408901:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309761262, at 4309761280 00000100:00000010:0.0:1788574401.408915:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab240. 00000020:00000001:0.0:1788574401.856991:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574401.856996:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574401.856999:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574402.176941:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574402.176948:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574402.176952:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574402.176960:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574402.176965:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574402.176968:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574402.176970:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574402.176972:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574402.176988:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574403.200911:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574403.200917:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574403.200921:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574403.200928:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574403.200935:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574403.200941:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574403.200946:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574403.200948:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574403.200949:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574403.905081:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574403.905088:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574403.905093:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574404.225051:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574404.225059:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574404.225064:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574404.225069:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574404.225075:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574404.225079:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574404.225083:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574404.225089:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574404.225097:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574405.248906:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574405.248911:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574405.248916:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574405.248921:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574405.248927:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574405.248932:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574405.248936:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574405.248938:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574405.248941:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574405.952980:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574405.952983:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574405.952986:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574406.272963:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574406.272970:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574406.272975:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574406.272983:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574406.272998:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574406.273002:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574406.273007:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574406.273012:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574406.273018:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574406.528903:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309766381, at 4309766400 00000100:00000010:0.0:1788574406.528915:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab0c0. 00000020:00000001:0.0:1788574406.977055:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574406.977060:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574406.977064:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574407.296924:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574407.296929:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574407.296932:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574407.296936:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574407.296939:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574407.296942:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574407.296947:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574407.296950:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574407.296955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574407.808939:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574407.808951:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574407.808955:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574407.808959:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574407.808964:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574407.808972:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15095 00000400:00000200:0.0:1788574407.808981:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574407.808986:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574407.808988:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574407.808991:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.808993:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574407.808995:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.808997:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574407.809000:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574407.809013:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574407.809016:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574407.809018:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574407.809020:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574407.809022:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574407.809024:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809026:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574407.809027:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809028:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574407.809030:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574407.809033:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809034:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574407.809036:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574407.809039:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574407.809041:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.809042:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574407.809044:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574407.809048:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574407.809051:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574407.809053:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574407.809055:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.809057:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574407.809059:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574407.809062:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574407.809068:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574407.809073:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574407.809080:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516693632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574407.809088:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574407.809091:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574407.809095:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516693632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574407.809126:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574407.809129:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574407.809131:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574407.809132:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574407.809134:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574407.809135:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574407.809138:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15095 00000400:00000200:0.0:1788574407.809141:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574407.809144:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574407.809146:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574407.809149:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809153:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574407.809156:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809159:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574407.809163:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574407.809164:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574407.809166:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574407.809167:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574407.809168:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574407.809171:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574407.809176:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809181:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574407.809185:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809189:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574407.809194:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574407.809199:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809204:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574407.809213:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574407.809222:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574407.809228:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.809232:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574407.809236:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574407.809241:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.809246:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574407.809253:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574407.809260:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574407.809267:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574407.809271:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.809275:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574407.809277:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574407.809282:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574407.809289:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574407.809298:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574407.809311:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516693760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574407.809326:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574407.809330:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574407.809338:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516693760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574407.809350:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574407.809354:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574407.809360:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574407.809363:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574407.809365:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574407.809369:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574407.809374:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15095 00000400:00000200:0.0:1788574407.809381:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574407.809391:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574407.809395:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574407.809407:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809412:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574407.809419:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809425:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574407.809431:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574407.809434:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574407.809440:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574407.809446:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574407.809453:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574407.809458:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574407.809468:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809473:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574407.809480:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809486:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574407.809493:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574407.809502:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809506:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574407.809510:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574407.809514:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574407.809520:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.809526:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574407.809531:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574407.809539:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.809544:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574407.809549:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574407.809555:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574407.809561:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574407.809564:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.809584:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574407.809589:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574407.809593:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574407.809602:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574407.809621:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574407.809637:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516693888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574407.809662:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574407.809666:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574407.809677:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516693888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574407.809694:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574407.809701:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574407.809712:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574407.809716:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574407.809720:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574407.809728:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574407.809736:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15095 00000400:00000200:0.0:1788574407.809748:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574407.809754:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574407.809758:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574407.809766:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809772:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574407.809776:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809779:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574407.809783:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574407.809786:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574407.809789:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574407.809792:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574407.809796:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574407.809800:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574407.809806:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809858:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574407.809870:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809874:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574407.809878:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574407.809883:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574407.809887:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574407.809890:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574407.809894:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574407.809898:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.809902:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574407.809907:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574407.809912:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.809914:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574407.809918:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574407.809924:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574407.809931:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574407.809936:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.809942:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574407.809945:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574407.809950:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574407.809959:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574407.809967:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574407.809979:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516694016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574407.809996:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574407.809998:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574407.810005:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516694016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574407.810018:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574407.810024:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574407.810028:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574407.810029:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574407.810038:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574407.810044:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574407.810049:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15095 00000400:00000200:0.0:1788574407.810053:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574407.810058:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574407.810061:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574407.810064:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.810068:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574407.810074:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.810081:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574407.810088:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574407.810093:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574407.810098:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574407.810104:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574407.810112:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574407.810119:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574407.810129:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.810135:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574407.810141:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574407.810145:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574407.810149:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574407.810156:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574407.810160:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574407.810163:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574407.810168:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574407.810170:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.810174:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574407.810177:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574407.810184:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.810187:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574407.810198:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574407.810202:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574407.810206:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574407.810207:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574407.810209:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574407.810211:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574407.810214:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574407.810221:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574407.810229:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574407.810241:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516694144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574407.810258:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574407.810264:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574407.810272:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516694144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574407.810294:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574407.810301:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15105) 00000100:00000001:0.0:1788574408.320921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574408.320927:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574408.320933:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574408.320940:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574408.320949:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574408.320953:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574408.320967:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574408.320972:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574408.320976:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574409.025171:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574409.025177:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574409.025181:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574409.344947:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574409.344957:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574409.344961:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574409.344969:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574409.344976:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574409.344981:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574409.344999:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574409.345002:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574409.345005:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574410.369002:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574410.369009:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574410.369012:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574410.369016:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574410.369021:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574410.369025:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574410.369047:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574410.369050:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574410.369053:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574411.073086:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574411.073092:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574411.073097:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574411.392904:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574411.392911:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574411.392916:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574411.392924:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574411.392931:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574411.392938:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574411.392943:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574411.392946:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574411.392951:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574411.648906:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309771501, at 4309771520 00000100:00000010:0.0:1788574411.648916:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab4e0. 00000020:00000001:0.0:1788574412.096976:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574412.096979:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574412.096981:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574412.416903:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574412.416913:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574412.416917:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574412.416932:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574412.416939:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574412.416945:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.416957:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574412.416969:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.416993:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.929267:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574412.929278:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574412.929284:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574412.929304:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516694272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.929345:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574412.929350:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574412.929357:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574412.929362:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574412.929371:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574412.929386:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516694272:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574412.929401:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574412.929412:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574412.929417:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574412.929429:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574412.929443:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574412.929447:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574412.929458:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574412.929464:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574412.929481:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574412.929491:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516694272, portal 25 00000100:00000001:0.0:1788574412.929495:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574412.929501:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788574412.929513:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516694272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.929558:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574412.929572:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574412.929587:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574412.929600:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516694272, offset 0 00000400:00000200:0.0:1788574412.929632:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574412.929646:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574412.929659:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574412.929685:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574412.929694:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574412.929706:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574412.929714:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574412.929729:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547076:547076:256:4294967295] 192.168.202.128@tcp LPNI seq info [546999:546999:8:4294967295] 00000400:00000200:0.0:1788574412.929740:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574412.929756:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574412.929768:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574412.929774:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574412.929782:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574412.929786:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574412.929843:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574412.929848:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574412.929866:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574412.929881:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574412.929887:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574412.929893:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574412.929900:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574412.929908:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516694272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.929925:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574412.929933:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516694272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.929942:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.929946:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574412.929955:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574412.929970:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574412.929976:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309776622 00000100:00000010:0.0:1788574412.929986:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788574412.929997:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574412.930015:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309776622 00000100:00000010:0.0:1788574412.930019:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50a20. 00000100:00000001:0.0:1788574412.930023:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574412.930030:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574412.930040:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.930050:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574412.930061:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574412/real 1788574412] req@ffff8fc4ac822d80 x1875440516694272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574412.930078:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574412.930082:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574412.930089:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574412.930095:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574412.930100:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574412.930106:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574412.930114:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516694272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574412.930128:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516694272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.930140:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574412.930144:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.930147:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574412.930151:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574412.930162:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516694272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.930186:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574412.930195:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516694272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.930224:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574412.930231:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.930236:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574412.930248:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.930252:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574412.930256:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574412.930266:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574412.930275:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574412.930281:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574412.930296:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516694272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574412.930321:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516694272:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574412.930331:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.930339:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574412.930348:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516694272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.930381:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574412.930385:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574412.930391:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574412.930403:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574412.930411:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574412.930417:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574412.930426:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574412.930437:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788574412.930449:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574412.930457:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574412.930461:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788574412.930475:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574412.930480:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.930485:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.930497:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574412.930506:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.930520:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574412.930531:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574412.930537:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574412.930549:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516694400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.930562:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574412.930568:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574412.930573:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574412.930578:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574412.930585:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574412.930596:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516694400:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574412.930615:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574412.930629:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574412.930633:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574412.930638:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574412.930648:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574412.930655:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574412.930662:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574412.930666:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574412.930672:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574412.930677:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516694400, portal 10 00000100:00000001:0.0:1788574412.930684:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574412.930691:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788574412.930706:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516694400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.930723:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574412.930736:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574412.930741:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574412.930751:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516694400, offset 0 00000400:00000200:0.0:1788574412.930762:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574412.930768:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574412.930775:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574412.930780:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574412.930787:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574412.930790:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574412.930796:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574412.930836:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547077:547077:256:4294967295] 192.168.202.128@tcp LPNI seq info [547000:547000:8:4294967295] 00000400:00000200:0.0:1788574412.930844:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574412.930861:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574412.930865:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574412.930869:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574412.930874:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574412.930876:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574412.930890:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574412.930892:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574412.930894:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574412.930896:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574412.930899:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574412.930900:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574412.930902:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574412.930905:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516694400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.930911:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574412.930913:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516694400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.930918:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.930919:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574412.930921:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574412.930924:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574412.930926:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574412.930928:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309772820 00000100:00000001:0.0:1788574412.930930:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309776622 00000100:00000001:0.0:1788574412.930932:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309772820 00000100:00000001:0.0:1788574412.930933:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309776622 00000100:00000001:0.0:1788574412.930934:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574412.930939:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.930940:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574412.930943:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574412/real 1788574412] req@ffff8fc4ac821f80 x1875440516694400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574412.930949:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574412.930950:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574412.930951:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574412.930953:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574412.930954:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574412.930955:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574412.930957:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516694400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574412.930963:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516694400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.930967:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574412.930968:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.930969:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574412.930970:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574412.930972:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516694400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.930976:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574412.930979:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516694400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.930983:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574412.930984:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.930985:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574412.930986:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.930987:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574412.930989:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574412.930991:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574412.930996:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574412.930997:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574412.931000:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516694400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574412.931005:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516694400:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574412.931008:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574412.931011:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516694528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931015:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574412.931016:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574412.931018:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574412.931019:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574412.931021:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574412.931023:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516694528:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574412.931025:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574412.931027:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574412.931028:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574412.931029:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574412.931032:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574412.931033:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574412.931035:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574412.931037:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574412.931039:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574412.931041:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516694528, portal 10 00000100:00000001:0.0:1788574412.931042:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574412.931043:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788574412.931048:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516694528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931052:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574412.931056:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574412.931058:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574412.931060:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516694528, offset 0 00000400:00000200:0.0:1788574412.931062:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574412.931065:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574412.931068:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574412.931070:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574412.931071:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574412.931073:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574412.931075:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574412.931077:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547078:547078:256:4294967295] 192.168.202.128@tcp LPNI seq info [547001:547001:8:4294967295] 00000400:00000200:0.0:1788574412.931080:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574412.931084:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574412.931087:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574412.931089:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574412.931091:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574412.931093:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574412.931100:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574412.931102:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574412.931104:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574412.931106:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574412.931108:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574412.931109:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574412.931111:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574412.931113:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516694528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931118:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574412.931120:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516694528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931125:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.931129:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574412.931131:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574412.931134:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574412.931136:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574412.931138:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309772827 00000100:00000001:0.0:1788574412.931139:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309776622 00000100:00000001:0.0:1788574412.931141:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309772827 00000100:00000001:0.0:1788574412.931142:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309776622 00000100:00000001:0.0:1788574412.931144:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574412.931145:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.931146:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574412.931148:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574412/real 1788574412] req@ffff8fc4ac823480 x1875440516694528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574412.931153:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574412.931154:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574412.931155:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574412.931157:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574412.931158:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574412.931159:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574412.931161:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516694528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574412.931166:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516694528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931170:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574412.931171:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931173:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574412.931174:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574412.931176:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516694528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931180:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574412.931185:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516694528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931190:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574412.931191:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931192:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574412.931193:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931194:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574412.931197:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574412.931199:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574412.931202:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574412.931204:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574412.931208:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516694528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574412.931217:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516694528:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574412.931221:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574412.931225:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516694656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931232:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574412.931234:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574412.931237:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574412.931239:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574412.931241:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574412.931243:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516694656:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574412.931247:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574412.931249:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574412.931251:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574412.931253:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574412.931255:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574412.931257:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574412.931265:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574412.931268:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574412.931272:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574412.931275:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516694656, portal 4 00000100:00000001:0.0:1788574412.931277:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574412.931279:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788574412.931285:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516694656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931293:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574412.931295:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574412.931298:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574412.931301:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516694656, offset 0 00000400:00000200:0.0:1788574412.931305:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574412.931309:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574412.931313:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574412.931315:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574412.931317:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574412.931319:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574412.931321:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574412.931323:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547079:547079:256:4294967295] 192.168.202.128@tcp LPNI seq info [547002:547002:8:4294967295] 00000400:00000200:0.0:1788574412.931326:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574412.931330:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574412.931334:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574412.931336:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574412.931338:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574412.931340:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574412.931348:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574412.931349:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574412.931351:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574412.931353:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574412.931356:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574412.931360:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574412.931363:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574412.931366:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516694656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931371:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574412.931373:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516694656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931378:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.931379:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574412.931381:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574412.931384:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574412.931386:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574412.931387:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309772967 00000100:00000001:0.0:1788574412.931389:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309776622 00000100:00000001:0.0:1788574412.931390:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309772967 00000100:00000001:0.0:1788574412.931392:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309776622 00000100:00000001:0.0:1788574412.931393:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574412.931394:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.931395:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574412.931398:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574412/real 1788574412] req@ffff8fc4ac822300 x1875440516694656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574412.931403:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574412.931404:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574412.931405:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574412.931406:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574412.931408:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574412.931409:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574412.931411:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516694656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574412.931416:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516694656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931423:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574412.931425:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931426:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574412.931427:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574412.931430:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516694656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931434:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574412.931437:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516694656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931441:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574412.931442:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931443:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574412.931444:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931445:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574412.931446:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574412.931448:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574412.931450:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574412.931451:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574412.931453:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516694656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574412.931458:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516694656:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574412.931461:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931462:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574412.931464:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516694400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931468:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574412.931469:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574412.931473:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574412.931475:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574412.931476:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574412.931477:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574412.931479:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574412.931480:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788574412.931482:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574412.931483:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574412.931484:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788574412.931486:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574412.931487:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931488:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574412.931491:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516694528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931495:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574412.931496:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574412.931497:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574412.931499:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574412.931499:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574412.931500:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574412.931501:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574412.931502:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574412.931503:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574412.931504:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574412.931505:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788574412.931506:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574412.931507:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931508:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574412.931510:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516694656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931513:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574412.931514:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574412.931515:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574412.931516:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574412.931517:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574412.931517:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574412.931521:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574412.931522:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788574412.931524:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574412.931524:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574412.931525:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788574412.931527:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574412.931527:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931529:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931531:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574412.931532:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.931534:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574412.931535:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574412.931536:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574412.931538:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516694784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931543:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574412.931544:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574412.931546:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574412.931547:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574412.931549:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574412.931550:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516694784:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574412.931552:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574412.931554:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574412.931555:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574412.931556:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574412.931558:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574412.931559:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574412.931562:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574412.931564:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574412.931568:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574412.931571:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516694784, portal 4 00000100:00000001:0.0:1788574412.931573:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574412.931574:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788574412.931580:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516694784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931598:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574412.931601:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574412.931603:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574412.931607:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516694784, offset 0 00000400:00000200:0.0:1788574412.931611:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574412.931615:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574412.931618:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574412.931635:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574412.931637:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574412.931639:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574412.931641:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574412.931643:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547080:547080:256:4294967295] 192.168.202.128@tcp LPNI seq info [547003:547003:8:4294967295] 00000400:00000200:0.0:1788574412.931646:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574412.931650:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574412.931653:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574412.931656:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56e00. 00000800:00000200:0.0:1788574412.931659:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574412.931660:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574412.931668:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574412.931669:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56e00. 00000400:00000200:0.0:1788574412.931671:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574412.931674:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574412.931676:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574412.931677:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574412.931679:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574412.931681:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516694784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931687:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574412.931689:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516694784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574437 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931701:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.931702:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574412.931705:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574412.931707:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574412.931710:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574412.931711:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309772969 00000100:00000001:0.0:1788574412.931713:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309776622 00000100:00000001:0.0:1788574412.931715:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309772969 00000100:00000001:0.0:1788574412.931716:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309776622 00000100:00000001:0.0:1788574412.931717:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574412.931718:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.931720:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574412.931722:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574412/real 1788574412] req@ffff8fc4ac820700 x1875440516694784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574412.931727:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574412.931728:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574412.931729:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574412.931731:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574412.931732:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574412.931733:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574412.931735:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516694784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574412.931740:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516694784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931744:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574412.931745:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931746:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574412.931747:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574412.931749:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516694784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931758:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574412.931761:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516694784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931766:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574412.931767:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931768:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574412.931769:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931771:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574412.931772:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574412.931774:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574412.931776:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574412.931777:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574412.931779:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516694784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574412.931784:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516694784:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574412.931786:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931788:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574412.931790:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516694784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574437 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574412.931794:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574412.931795:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574412.931796:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574412.931797:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574412.931798:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574412.931799:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574412.931800:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574412.931801:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788574412.931803:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574412.931806:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574412.931823:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788574412.931826:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574412.931827:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931828:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574412.931831:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574412.931832:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574412.931833:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574412.931834:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574413.440917:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574413.440921:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574413.440924:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574413.440931:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574413.440942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574413.440952:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574413.440991:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574413.440999:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574413.441015:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574414.144959:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574414.144963:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574414.144966:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574414.464910:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574414.464919:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574414.464923:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574414.464928:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574414.464934:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574414.464941:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574414.464953:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574414.464957:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574414.464963:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574415.488938:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574415.488950:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574415.488956:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574415.488966:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574415.488971:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574415.488975:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574415.488983:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574415.488987:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574415.489003:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574416.192963:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574416.192966:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574416.192969:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574416.512910:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574416.512918:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574416.512920:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574416.512926:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574416.512931:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574416.512935:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574416.512940:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574416.512944:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574416.512955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574416.768905:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309776622, at 4309776640 00000100:00000010:0.0:1788574416.768924:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b503c0. 00000020:00000001:0.0:1788574417.216980:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574417.216983:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574417.216986:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574417.536961:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574417.536968:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574417.536971:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574417.536987:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574417.536994:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574417.537000:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574417.537008:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574417.537012:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574417.537015:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574418.048914:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574418.048925:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574418.048928:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574418.048933:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574418.048938:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574418.048947:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15105 00000400:00000200:0.0:1788574418.048957:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574418.048964:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574418.048988:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574418.048996:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049003:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574418.049010:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049015:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574418.049021:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574418.049024:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574418.049027:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574418.049031:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574418.049034:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574418.049037:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574418.049042:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049045:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574418.049049:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049052:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574418.049056:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574418.049060:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049064:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574418.049068:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574418.049072:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574418.049076:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.049079:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574418.049081:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574418.049089:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574418.049092:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574418.049097:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574418.049099:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.049102:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574418.049105:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574418.049110:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574418.049118:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574418.049126:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574418.049142:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516694912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574418.049152:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574418.049157:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574418.049163:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516694912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574418.049184:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574418.049188:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574418.049192:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574418.049194:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574418.049198:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574418.049201:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574418.049203:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15105 00000400:00000200:0.0:1788574418.049208:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574418.049214:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574418.049218:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574418.049224:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049228:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574418.049234:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049240:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574418.049246:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574418.049248:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574418.049252:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574418.049256:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574418.049259:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574418.049265:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574418.049272:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049278:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574418.049283:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049294:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574418.049300:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574418.049309:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049316:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574418.049320:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574418.049327:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574418.049331:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.049338:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574418.049341:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574418.049344:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.049349:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574418.049355:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574418.049360:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574418.049364:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574418.049366:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.049368:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574418.049369:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574418.049371:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574418.049374:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574418.049380:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574418.049391:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516695040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574418.049402:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574418.049406:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574418.049413:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516695040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574418.049426:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574418.049432:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574418.049439:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574418.049445:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574418.049450:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574418.049466:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574418.049474:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15105 00000400:00000200:0.0:1788574418.049488:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574418.049497:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574418.049506:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574418.049516:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049530:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574418.049537:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049542:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574418.049548:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574418.049552:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574418.049557:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574418.049561:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574418.049564:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574418.049567:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574418.049571:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049574:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574418.049580:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049584:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574418.049592:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574418.049601:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049616:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574418.049623:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574418.049629:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574418.049634:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.049639:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574418.049643:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574418.049647:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.049651:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574418.049658:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574418.049690:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574418.049698:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574418.049699:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.049702:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574418.049704:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574418.049707:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574418.049715:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574418.049723:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574418.049735:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516695168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574418.049760:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574418.049765:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574418.049776:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516695168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574418.049789:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574418.049793:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574418.049799:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574418.049802:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574418.049805:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574418.049828:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574418.049831:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15105 00000400:00000200:0.0:1788574418.049837:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574418.049843:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574418.049845:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574418.049850:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049855:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574418.049860:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049863:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574418.049869:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574418.049870:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574418.049877:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574418.049879:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574418.049881:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574418.049884:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574418.049892:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049896:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574418.049900:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049903:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574418.049908:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574418.049912:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574418.049915:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574418.049919:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574418.049923:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574418.049928:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.049931:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574418.049935:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574418.049939:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.049941:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574418.049946:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574418.049950:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574418.049954:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574418.049957:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.049960:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574418.049962:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574418.049965:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574418.049971:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574418.049978:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574418.049985:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516695296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574418.049993:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574418.049996:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574418.050005:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516695296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574418.050033:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574418.050037:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574418.050044:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574418.050049:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574418.050052:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574418.050058:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574418.050062:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15105 00000400:00000200:0.0:1788574418.050068:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574418.050071:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574418.050072:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574418.050077:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.050080:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574418.050086:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.050090:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574418.050094:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574418.050096:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574418.050097:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574418.050098:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574418.050100:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574418.050103:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574418.050110:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.050113:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574418.050118:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574418.050122:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574418.050128:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574418.050134:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574418.050138:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574418.050150:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574418.050155:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574418.050159:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.050164:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574418.050169:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574418.050173:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.050178:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574418.050184:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574418.050189:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574418.050193:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574418.050196:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574418.050201:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574418.050203:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574418.050206:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574418.050214:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574418.050221:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574418.050233:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516695424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574418.050257:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574418.050262:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574418.050273:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516695424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574418.050301:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574418.050309:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15115) 00000100:00000001:0.0:1788574418.560882:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574418.560888:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574418.560892:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574418.560900:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574418.560905:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574418.560911:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574418.560918:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574418.560922:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574418.560925:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574419.264959:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574419.264970:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574419.264974:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574420.608952:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574420.608961:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574420.608967:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574420.608976:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574420.608984:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574420.608989:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574420.608999:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574420.609003:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574420.609010:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574421.312992:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574421.312995:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574421.312997:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574421.632958:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574421.632962:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574421.632965:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574421.632968:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574421.632974:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574421.632979:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574421.632998:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574421.633002:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574421.633006:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574421.888920:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309781741, at 4309781760 00000100:00000010:0.0:1788574421.888951:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab000. 00000020:00000001:0.0:1788574422.337004:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574422.337010:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574422.337014:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574424.385072:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574424.385077:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574424.385081:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574424.704927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574424.704932:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574424.704935:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574424.704939:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574424.704945:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574424.704950:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574424.704977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574424.704981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574424.704984:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574425.728953:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574425.728963:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574425.728968:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574425.728978:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574425.728986:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574425.728991:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574425.729027:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574425.729029:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574425.729032:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574426.433028:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574426.433031:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574426.433033:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574426.752926:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574426.752934:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574426.752937:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574426.752943:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574426.752950:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574426.752957:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574426.752971:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574426.752981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574426.752995:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574427.008891:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309786861, at 4309786880 00000100:00000010:0.0:1788574427.008907:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab3c0. 00000020:00000001:0.0:1788574427.456989:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574427.456993:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574427.456995:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574427.776921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574427.776930:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574427.776935:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574427.776943:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574427.776960:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574427.776965:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574427.777011:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574427.777020:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574427.777032:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574428.288974:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574428.288987:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574428.288990:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574428.288994:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574428.288999:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574428.289005:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15115 00000400:00000200:0.0:1788574428.289013:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574428.289018:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574428.289020:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574428.289023:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289025:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574428.289027:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289029:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574428.289031:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574428.289033:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574428.289035:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574428.289037:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574428.289038:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574428.289040:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574428.289042:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289043:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574428.289045:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289046:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574428.289048:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574428.289051:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289052:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574428.289054:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574428.289057:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574428.289059:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289077:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574428.289079:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574428.289085:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574428.289089:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574428.289092:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574428.289093:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289095:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574428.289098:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574428.289102:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574428.289108:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574428.289113:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574428.289120:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516696192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574428.289127:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574428.289131:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574428.289134:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516696192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574428.289150:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574428.289153:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574428.289155:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574428.289156:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574428.289158:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574428.289159:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574428.289161:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15115 00000400:00000200:0.0:1788574428.289164:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574428.289166:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574428.289168:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574428.289170:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289172:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574428.289174:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289176:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574428.289182:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574428.289184:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574428.289186:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574428.289189:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574428.289192:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574428.289195:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574428.289200:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289202:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574428.289203:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289204:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574428.289208:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574428.289212:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289217:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574428.289221:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574428.289228:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574428.289231:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289235:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574428.289238:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574428.289244:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289248:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574428.289253:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574428.289258:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574428.289260:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574428.289263:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289268:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574428.289272:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574428.289273:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574428.289277:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574428.289284:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574428.289293:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516696320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574428.289302:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574428.289309:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574428.289315:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516696320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574428.289323:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574428.289326:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574428.289331:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574428.289333:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574428.289335:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574428.289340:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574428.289345:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15115 00000400:00000200:0.0:1788574428.289352:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574428.289360:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574428.289363:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574428.289369:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289371:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574428.289375:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289377:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574428.289379:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574428.289381:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574428.289383:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574428.289386:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574428.289387:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574428.289389:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574428.289394:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289398:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574428.289400:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289402:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574428.289403:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574428.289406:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289414:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574428.289417:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574428.289420:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574428.289424:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289425:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574428.289427:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574428.289428:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289429:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574428.289432:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574428.289433:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574428.289435:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574428.289436:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289439:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574428.289441:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574428.289442:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574428.289447:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574428.289453:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574428.289463:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516696448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574428.289478:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574428.289480:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574428.289487:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516696448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574428.289496:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574428.289501:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574428.289504:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574428.289507:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574428.289510:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574428.289512:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574428.289515:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15115 00000400:00000200:0.0:1788574428.289521:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574428.289533:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574428.289537:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574428.289555:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289563:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574428.289567:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289573:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574428.289578:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574428.289582:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574428.289586:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574428.289589:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574428.289592:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574428.289597:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574428.289601:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289604:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574428.289607:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289609:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574428.289612:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574428.289617:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289621:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574428.289625:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574428.289629:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574428.289633:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289634:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574428.289637:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574428.289639:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289640:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574428.289643:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574428.289646:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574428.289648:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574428.289650:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289652:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574428.289660:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574428.289664:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574428.289668:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574428.289674:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574428.289680:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516696576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574428.289690:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574428.289694:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574428.289700:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516696576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574428.289710:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574428.289712:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574428.289715:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574428.289719:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574428.289720:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574428.289723:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574428.289726:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15115 00000400:00000200:0.0:1788574428.289731:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574428.289738:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574428.289740:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574428.289745:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289747:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574428.289751:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289754:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574428.289757:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574428.289759:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574428.289762:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574428.289764:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574428.289766:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574428.289768:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574428.289792:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289795:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574428.289798:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289802:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574428.289832:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574428.289837:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574428.289840:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574428.289843:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574428.289847:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574428.289853:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289855:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574428.289857:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574428.289861:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289864:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574428.289867:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574428.289871:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574428.289877:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574428.289880:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574428.289883:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574428.289884:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574428.289889:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574428.289893:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574428.289898:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574428.289907:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516696704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574428.289916:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574428.289920:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574428.289928:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516696704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574428.289938:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574428.289944:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15125) 00000100:00000001:0.0:1788574428.801043:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574428.801047:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574428.801049:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574428.801053:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574428.801058:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574428.801062:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574428.801126:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574428.801129:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574428.801131:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574429.504958:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574429.504961:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574429.504963:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574429.824929:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574429.824934:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574429.824939:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574429.824945:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574429.824951:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574429.824956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574429.824961:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574429.824963:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574429.824966:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574430.848906:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574430.848913:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574430.848916:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574430.848923:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574430.848928:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574430.848932:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574430.848937:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574430.848941:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574430.848945:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574431.553000:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574431.553004:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574431.553006:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574431.873079:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574431.873085:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574431.873088:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574431.873092:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574431.873095:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574431.873111:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574431.873113:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574431.873115:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574431.873117:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574432.128928:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309791981, at 4309792000 00000100:00000010:0.0:1788574432.128946:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab420. 00000020:00000001:0.0:1788574432.577069:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574432.577072:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574432.577075:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574432.896935:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574432.896941:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574432.896946:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574432.896955:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574432.896971:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574432.896976:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574432.896989:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574432.896993:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574432.896998:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574433.408938:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574433.408948:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574433.408951:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574433.408956:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574433.408961:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574433.408969:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15120 00000400:00000200:0.0:1788574433.408976:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574433.408981:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574433.408983:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574433.408986:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.408995:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574433.408997:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.408999:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574433.409002:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574433.409004:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574433.409020:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574433.409026:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574433.409028:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574433.409030:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574433.409033:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409035:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574433.409036:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409037:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574433.409039:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574433.409042:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409043:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574433.409045:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574433.409048:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574433.409050:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.409052:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574433.409053:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574433.409058:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574433.409061:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574433.409064:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574433.409066:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.409068:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574433.409070:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574433.409074:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574433.409080:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574433.409086:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574433.409093:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516696832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574433.409101:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574433.409104:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574433.409108:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516696832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574433.409123:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574433.409134:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574433.409138:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574433.409139:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574433.409141:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574433.409143:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574433.409146:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15120 00000400:00000200:0.0:1788574433.409150:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574433.409153:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574433.409154:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574433.409157:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409162:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574433.409164:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409168:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574433.409171:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574433.409174:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574433.409176:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574433.409180:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574433.409184:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574433.409188:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574433.409195:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409199:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574433.409204:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409208:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574433.409214:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574433.409219:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409223:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574433.409226:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574433.409232:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574433.409237:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.409243:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574433.409252:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574433.409258:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.409260:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574433.409265:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574433.409268:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574433.409273:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574433.409274:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.409275:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574433.409277:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574433.409281:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574433.409287:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574433.409292:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574433.409302:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516696960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574433.409322:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574433.409325:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574433.409332:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516696960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574433.409352:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574433.409358:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574433.409365:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574433.409369:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574433.409374:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574433.409381:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574433.409394:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15120 00000400:00000200:0.0:1788574433.409410:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574433.409416:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574433.409419:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574433.409426:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409430:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574433.409434:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409445:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574433.409447:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574433.409448:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574433.409450:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574433.409453:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574433.409457:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574433.409463:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574433.409469:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409471:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574433.409476:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409478:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574433.409480:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574433.409487:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409489:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574433.409491:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574433.409496:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574433.409500:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.409502:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574433.409506:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574433.409516:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.409520:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574433.409525:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574433.409530:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574433.409534:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574433.409538:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.409541:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574433.409543:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574433.409545:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574433.409549:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574433.409552:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574433.409558:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516697088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574433.409598:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574433.409603:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574433.409613:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516697088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574433.409638:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574433.409643:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574433.409647:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574433.409651:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574433.409654:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574433.409657:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574433.409663:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15120 00000400:00000200:0.0:1788574433.409671:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574433.409674:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574433.409676:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574433.409682:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409684:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574433.409688:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409691:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574433.409694:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574433.409697:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574433.409698:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574433.409700:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574433.409703:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574433.409708:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574433.409715:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409719:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574433.409725:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409730:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574433.409740:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574433.409749:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409755:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574433.409759:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574433.409765:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574433.409771:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.409777:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574433.409781:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574433.409786:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.409788:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574433.409792:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574433.409798:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574433.409805:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574433.409830:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.409833:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574433.409835:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574433.409851:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574433.409857:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574433.409862:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574433.409869:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516697216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574433.409888:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574433.409893:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574433.409902:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516697216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574433.409912:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574433.409919:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574433.409926:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574433.409930:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574433.409934:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574433.409941:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574433.409956:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15120 00000400:00000200:0.0:1788574433.409965:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574433.409972:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574433.409979:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574433.409987:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.409993:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574433.410000:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.410005:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574433.410012:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574433.410016:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574433.410022:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574433.410028:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574433.410033:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574433.410040:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574433.410049:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.410053:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574433.410058:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574433.410063:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574433.410069:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574433.410076:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574433.410082:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574433.410086:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574433.410090:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574433.410096:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.410101:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574433.410105:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574433.410113:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.410118:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574433.410126:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574433.410133:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574433.410138:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574433.410149:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574433.410155:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574433.410160:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574433.410164:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574433.410172:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574433.410180:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574433.410198:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516697344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574433.410217:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574433.410222:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574433.410232:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516697344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574433.410256:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574433.410265:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15130) 00000100:00000001:0.0:1788574433.920933:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574433.920941:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574433.920945:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574433.920953:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574433.920963:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574433.920969:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574433.920986:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574433.920990:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574433.920994:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574434.625033:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574434.625036:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574434.625038:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574434.944900:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574434.944903:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574434.944906:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574434.944908:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574434.944911:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574434.944913:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574434.944928:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574434.944932:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574434.944938:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574434.944985:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574434.944989:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574434.944991:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574434.944994:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574434.944996:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574434.944999:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574434.945002:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574434.945004:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574434.945006:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574435.968951:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574435.968956:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574435.968958:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574435.968965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574435.968972:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574435.968976:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574435.968988:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574435.968991:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574435.968993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574436.481524:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574436.481529:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574436.481533:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574436.481543:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574433/real 1788574436] req@ffff8fc4b80c3b80 x1875440516697344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574436.481556:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574436.481559:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574436.481562:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a07e8. 00000400:00000200:0.0:1788574436.481566:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed6e8 00000400:00000010:0.0:1788574436.481568:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed6e8. 00000100:00000001:0.0:1788574436.481572:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574436.481576:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c3b80 x1875440516697344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574436.481587:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c3b80 x1875440516697344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.481596:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574436.481599:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.481610:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574436.481612:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574436.481618:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c3b80 x1875440516697344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.481627:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574436.481632:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c3b80 x1875440516697344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.481641:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574436.481645:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.481646:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574436.481648:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.481650:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574436.481654:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574436.481659:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574436.481665:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574436.481668:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574436.481672:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c3b80 x1875440516697344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574436.481680:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516697344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574436.481687:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574436.481691:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574433/real 1788574436] req@ffff8fc4b80c1500 x1875440516697088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574436.481700:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574436.481701:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574436.481703:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a0790. 00000400:00000200:0.0:1788574436.481705:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed198 00000400:00000010:0.0:1788574436.481708:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed198. 00000100:00000001:0.0:1788574436.481710:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574436.481720:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c1500 x1875440516697088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574436.481728:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c1500 x1875440516697088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.481734:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574436.481736:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.481738:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574436.481740:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574436.481743:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c1500 x1875440516697088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.481749:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574436.481753:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c1500 x1875440516697088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.481759:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574436.481761:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.481763:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574436.481764:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.481766:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574436.481769:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574436.481771:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574436.481775:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574436.481777:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574436.481780:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c1500 x1875440516697088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574436.481788:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516697088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574436.481792:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574436.481796:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574433/real 1788574436] req@ffff8fc4b80c1180 x1875440516697216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574436.481831:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574436.481833:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574436.481836:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a06e0. 00000400:00000200:0.0:1788574436.481839:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed990 00000400:00000010:0.0:1788574436.481841:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed990. 00000100:00000001:0.0:1788574436.481843:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574436.481847:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c1180 x1875440516697216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574436.481856:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c1180 x1875440516697216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.481862:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574436.481864:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.481866:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574436.481867:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574436.481886:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c1180 x1875440516697216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.481893:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574436.481898:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c1180 x1875440516697216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.481905:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574436.481908:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.481910:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574436.481912:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.481914:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574436.481916:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574436.481919:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574436.481922:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574436.481925:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574436.481935:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c1180 x1875440516697216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574436.481943:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516697216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574436.481960:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574436.481964:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574433/real 1788574436] req@ffff8fc4b80c0380 x1875440516696960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574436.481972:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574436.481973:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574436.481975:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a04d0. 00000400:00000200:0.0:1788574436.481978:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed110 00000400:00000010:0.0:1788574436.481980:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed110. 00000100:00000001:0.0:1788574436.481983:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574436.481985:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c0380 x1875440516696960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574436.481993:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c0380 x1875440516696960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.481999:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574436.482001:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482003:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574436.482004:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574436.482008:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c0380 x1875440516696960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.482013:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574436.482017:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c0380 x1875440516696960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.482022:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574436.482025:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482034:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574436.482035:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482037:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574436.482039:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574436.482041:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574436.482045:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574436.482047:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574436.482051:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c0380 x1875440516696960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574436.482058:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516696960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574436.482063:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574436.482067:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574433/real 1788574436] req@ffff8fc4b80c0a80 x1875440516696832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574436.482074:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574436.482076:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574436.482079:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a0210. 00000400:00000200:0.0:1788574436.482082:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed770 00000400:00000010:0.0:1788574436.482084:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed770. 00000100:00000001:0.0:1788574436.482087:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574436.482090:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c0a80 x1875440516696832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574436.482097:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c0a80 x1875440516696832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.482103:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574436.482104:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482106:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574436.482107:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574436.482111:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c0a80 x1875440516696832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.482129:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574436.482134:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c0a80 x1875440516696832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.482141:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574436.482144:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482146:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574436.482147:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482149:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574436.482151:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574436.482153:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574436.482158:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574436.482161:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574436.482167:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c0a80 x1875440516696832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574436.482174:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516696832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574436.482180:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482182:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574436.482185:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c3b80 x1875440516697344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.482192:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574436.482195:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574436.482199:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788574436.482204:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574436.482207:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574436.482213:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574436.482216:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574436.482219:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574436.482223:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574436.482231:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574436.482233:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c3b80. 00000100:00000001:0.0:1788574436.482237:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574436.482239:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482241:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574436.482245:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c1500 x1875440516697088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.482253:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574436.482254:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574436.482256:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788574436.482258:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574436.482260:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574436.482261:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574436.482263:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574436.482265:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574436.482268:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574436.482269:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574436.482270:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c1500. 00000100:00000001:0.0:1788574436.482273:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574436.482274:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482276:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574436.482280:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c1180 x1875440516697216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.482287:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574436.482289:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574436.482290:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788574436.482293:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574436.482294:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574436.482295:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574436.482297:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574436.482299:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574436.482301:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574436.482302:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574436.482304:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c1180. 00000100:00000001:0.0:1788574436.482306:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574436.482307:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482315:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574436.482319:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c0380 x1875440516696960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.482327:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574436.482328:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574436.482330:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788574436.482333:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574436.482334:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574436.482335:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574436.482337:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574436.482339:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574436.482340:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574436.482341:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574436.482342:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c0380. 00000100:00000001:0.0:1788574436.482349:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574436.482350:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482351:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574436.482354:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c0a80 x1875440516696832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574458 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574436.482359:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574436.482359:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574436.482360:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788574436.482362:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574436.482363:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574436.482363:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574436.482364:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574436.482365:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574436.482367:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574436.482368:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574436.482369:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c0a80. 00000100:00000001:0.0:1788574436.482370:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574436.482371:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482373:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.482380:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574436.482382:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574436.482384:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574436.482393:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574436.673016:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574436.673019:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574436.673022:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574436.992926:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574436.992933:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574436.992937:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574436.992944:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574436.992957:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574436.992962:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.992990:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574436.992994:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574436.992996:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574437.248919:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309797101, at 4309797120 00000100:00000010:0.0:1788574437.248934:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abba0. 00000020:00000001:0.0:1788574437.697022:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574437.697025:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574437.697028:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574438.017006:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574438.017011:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574438.017013:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574438.017017:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574438.017021:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574438.017023:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.017026:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574438.017028:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.017031:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.529160:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574438.529166:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574438.529171:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574438.529184:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516697472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.529204:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574438.529208:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574438.529215:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574438.529220:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574438.529227:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574438.529243:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516697472:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574438.529253:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574438.529259:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574438.529265:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574438.529272:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574438.529279:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574438.529282:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574438.529289:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574438.529294:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574438.529303:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574438.529313:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516697472, portal 25 00000100:00000001:0.0:1788574438.529323:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574438.529327:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788574438.529338:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516697472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.529355:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574438.529362:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574438.529367:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574438.529374:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516697472, offset 0 00000400:00000200:0.0:1788574438.529390:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574438.529404:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574438.529415:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574438.529425:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574438.529434:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574438.529443:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574438.529452:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574438.529463:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547101:547101:256:4294967295] 192.168.202.128@tcp LPNI seq info [547024:547024:8:4294967295] 00000400:00000200:0.0:1788574438.529485:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574438.529506:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574438.529521:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574438.529530:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574438.529550:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574438.529558:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574438.529583:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574438.529589:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574438.529597:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574438.529606:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574438.529615:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574438.529621:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574438.529629:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574438.529640:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516697472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.529661:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574438.529669:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516697472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.529678:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.529680:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574438.529687:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574438.529691:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574438.529694:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309802221 00000100:00000010:0.0:1788574438.529700:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574438.529707:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574438.529714:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309802221 00000100:00000010:0.0:1788574438.529717:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b501e0. 00000100:00000001:0.0:1788574438.529721:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574438.529725:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574438.529728:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.529731:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574438.529736:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574438/real 1788574438] req@ffff8fc4901a7800 x1875440516697472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574438.529747:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574438.529756:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574438.529761:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574438.529767:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574438.529771:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574438.529795:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574438.529804:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516697472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574438.529853:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516697472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.529869:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574438.529874:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.529880:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574438.529885:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574438.529909:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516697472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.529927:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574438.529939:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516697472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.529958:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574438.529965:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.529971:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574438.529974:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.529979:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574438.529983:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574438.529989:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574438.529997:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574438.530002:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574438.530011:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516697472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574438.530031:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516697472:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574438.530049:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.530054:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574438.530062:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516697472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530077:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574438.530081:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574438.530086:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574438.530093:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574438.530097:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574438.530100:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574438.530104:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574438.530108:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788574438.530116:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574438.530120:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574438.530124:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788574438.530132:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574438.530135:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.530139:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.530150:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574438.530154:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.530159:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574438.530164:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574438.530167:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574438.530176:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516697600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530189:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574438.530193:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574438.530199:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574438.530204:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574438.530209:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574438.530217:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516697600:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574438.530225:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574438.530231:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574438.530234:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574438.530247:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574438.530253:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574438.530257:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574438.530268:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574438.530272:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574438.530278:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574438.530285:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516697600, portal 10 00000100:00000001:0.0:1788574438.530290:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574438.530293:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788574438.530306:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516697600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530320:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574438.530327:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574438.530332:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574438.530336:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516697600, offset 0 00000400:00000200:0.0:1788574438.530347:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574438.530356:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574438.530364:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574438.530368:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574438.530374:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574438.530381:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574438.530386:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574438.530392:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547102:547102:256:4294967295] 192.168.202.128@tcp LPNI seq info [547025:547025:8:4294967295] 00000400:00000200:0.0:1788574438.530402:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574438.530413:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574438.530419:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574438.530423:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574438.530428:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574438.530431:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574438.530446:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574438.530449:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574438.530459:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574438.530465:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574438.530470:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574438.530472:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574438.530476:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574438.530480:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516697600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530490:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574438.530494:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516697600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530501:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.530503:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574438.530507:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574438.530511:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574438.530515:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574438.530518:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309798419 00000100:00000001:0.0:1788574438.530521:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309802221 00000100:00000001:0.0:1788574438.530523:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309798419 00000100:00000001:0.0:1788574438.530525:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309802221 00000100:00000001:0.0:1788574438.530528:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574438.530530:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.530533:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574438.530537:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574438/real 1788574438] req@ffff8fc4901a5c00 x1875440516697600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574438.530546:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574438.530548:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574438.530550:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574438.530553:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574438.530554:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574438.530557:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574438.530569:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516697600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574438.530579:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516697600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530587:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574438.530589:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.530591:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574438.530593:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574438.530596:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516697600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530601:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574438.530603:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516697600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530608:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574438.530610:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.530612:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574438.530613:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.530615:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574438.530616:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574438.530619:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574438.530622:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574438.530625:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574438.530629:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516697600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574438.530637:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516697600:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574438.530641:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574438.530645:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516697728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530677:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574438.530679:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574438.530682:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574438.530684:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574438.530686:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574438.530690:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516697728:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574438.530694:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574438.530696:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574438.530698:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574438.530701:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574438.530705:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574438.530707:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574438.530710:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574438.530712:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574438.530714:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574438.530716:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516697728, portal 10 00000100:00000001:0.0:1788574438.530718:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574438.530719:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788574438.530724:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516697728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530728:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574438.530730:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574438.530731:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574438.530733:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516697728, offset 0 00000400:00000200:0.0:1788574438.530735:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574438.530738:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574438.530741:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574438.530743:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574438.530745:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574438.530747:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574438.530748:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574438.530755:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547103:547103:256:4294967295] 192.168.202.128@tcp LPNI seq info [547026:547026:8:4294967295] 00000400:00000200:0.0:1788574438.530758:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574438.530763:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574438.530766:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574438.530768:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574438.530771:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574438.530773:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574438.530780:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574438.530781:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574438.530784:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574438.530786:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574438.530788:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574438.530789:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574438.530791:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574438.530794:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516697728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530799:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574438.530801:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516697728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530806:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.530825:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574438.530828:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574438.530830:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574438.530833:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574438.530834:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309798427 00000100:00000001:0.0:1788574438.530836:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309802221 00000100:00000001:0.0:1788574438.530837:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309798427 00000100:00000001:0.0:1788574438.530839:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309802221 00000100:00000001:0.0:1788574438.530840:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574438.530846:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.530848:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574438.530853:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574438/real 1788574438] req@ffff8fc4901a5180 x1875440516697728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574438.530862:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574438.530863:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574438.530865:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574438.530868:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574438.530869:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574438.530872:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574438.530875:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516697728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574438.530884:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516697728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530890:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574438.530891:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.530906:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574438.530907:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574438.530912:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516697728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530919:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574438.530923:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516697728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530930:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574438.530933:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.530934:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574438.530936:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.530937:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574438.530940:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574438.530943:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574438.530953:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574438.530955:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574438.530960:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516697728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574438.530967:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516697728:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574438.530970:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574438.530972:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516697856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.530976:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574438.530978:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574438.530979:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574438.530981:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574438.530982:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574438.530984:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516697856:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574438.530986:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574438.530987:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574438.530989:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574438.530990:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574438.530993:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574438.530994:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574438.530997:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574438.531000:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574438.531004:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574438.531007:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516697856, portal 4 00000100:00000001:0.0:1788574438.531009:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574438.531010:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788574438.531017:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516697856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531024:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574438.531032:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574438.531035:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574438.531037:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516697856, offset 0 00000400:00000200:0.0:1788574438.531042:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574438.531046:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574438.531050:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574438.531052:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574438.531053:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574438.531055:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574438.531057:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574438.531059:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547104:547104:256:4294967295] 192.168.202.128@tcp LPNI seq info [547027:547027:8:4294967295] 00000400:00000200:0.0:1788574438.531062:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574438.531066:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574438.531070:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574438.531072:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574438.531075:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574438.531077:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574438.531084:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574438.531085:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574438.531087:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574438.531090:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574438.531092:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574438.531093:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574438.531095:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574438.531097:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516697856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531103:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574438.531105:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516697856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531109:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.531110:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574438.531118:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574438.531120:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574438.531122:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574438.531124:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309798567 00000100:00000001:0.0:1788574438.531125:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309802221 00000100:00000001:0.0:1788574438.531127:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309798567 00000100:00000001:0.0:1788574438.531129:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309802221 00000100:00000001:0.0:1788574438.531130:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574438.531131:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.531132:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574438.531135:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574438/real 1788574438] req@ffff8fc4901a7100 x1875440516697856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574438.531140:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574438.531141:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574438.531142:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574438.531144:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574438.531145:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574438.531146:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574438.531148:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516697856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574438.531154:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516697856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531158:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574438.531159:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531160:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574438.531160:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574438.531163:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516697856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531167:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574438.531169:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516697856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531177:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574438.531178:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531180:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574438.531180:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531182:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574438.531183:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574438.531184:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574438.531186:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574438.531188:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574438.531192:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516697856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574438.531200:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516697856:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574438.531204:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531206:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574438.531209:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516697600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531216:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574438.531218:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574438.531220:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574438.531223:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574438.531224:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574438.531226:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574438.531228:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574438.531230:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788574438.531233:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574438.531235:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574438.531237:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788574438.531242:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574438.531244:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531251:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574438.531254:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516697728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531258:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574438.531259:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574438.531260:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574438.531263:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574438.531264:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574438.531265:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574438.531266:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574438.531267:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788574438.531268:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574438.531269:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574438.531270:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788574438.531272:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574438.531273:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531274:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574438.531276:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516697856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531283:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574438.531284:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574438.531285:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574438.531288:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574438.531289:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574438.531290:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574438.531292:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574438.531293:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788574438.531296:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574438.531297:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574438.531298:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788574438.531302:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574438.531303:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531305:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531315:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574438.531317:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.531319:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574438.531326:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574438.531327:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574438.531332:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516697984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531339:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574438.531341:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574438.531343:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574438.531345:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574438.531347:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574438.531350:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516697984:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574438.531358:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574438.531360:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574438.531362:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574438.531364:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574438.531367:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574438.531369:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574438.531373:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574438.531376:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574438.531379:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574438.531382:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516697984, portal 4 00000100:00000001:0.0:1788574438.531383:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574438.531384:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788574438.531388:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516697984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531393:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574438.531394:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574438.531396:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574438.531397:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516697984, offset 0 00000400:00000200:0.0:1788574438.531400:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574438.531403:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574438.531405:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574438.531407:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574438.531409:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574438.531414:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574438.531416:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574438.531418:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547105:547105:256:4294967295] 192.168.202.128@tcp LPNI seq info [547028:547028:8:4294967295] 00000400:00000200:0.0:1788574438.531421:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574438.531426:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574438.531431:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574438.531434:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574438.531438:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574438.531440:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574438.531450:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574438.531452:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574438.531455:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574438.531459:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574438.531462:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574438.531464:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574438.531473:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574438.531477:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516697984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531485:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574438.531489:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516697984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574463 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531496:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.531498:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574438.531502:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574438.531506:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574438.531510:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574438.531512:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309798569 00000100:00000001:0.0:1788574438.531514:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309802221 00000100:00000001:0.0:1788574438.531517:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309798569 00000100:00000001:0.0:1788574438.531525:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309802221 00000100:00000001:0.0:1788574438.531528:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574438.531530:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.531532:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574438.531537:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574438/real 1788574438] req@ffff8fc4901a4a80 x1875440516697984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574438.531544:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574438.531546:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574438.531548:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574438.531550:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574438.531552:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574438.531555:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574438.531563:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516697984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574438.531572:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516697984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531579:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574438.531580:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531582:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574438.531584:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574438.531588:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516697984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531596:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574438.531598:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516697984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531603:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574438.531605:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531606:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574438.531607:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531612:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574438.531614:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574438.531616:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574438.531618:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574438.531620:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574438.531623:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516697984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574438.531628:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516697984:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574438.531631:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531632:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574438.531634:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516697984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574463 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574438.531638:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574438.531639:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574438.531640:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574438.531642:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574438.531642:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574438.531643:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574438.531644:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574438.531645:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788574438.531647:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574438.531648:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574438.531649:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788574438.531652:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574438.531653:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531654:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574438.531657:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574438.531658:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574438.531659:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574438.531660:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574439.040946:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574439.040951:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574439.040971:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574439.040977:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574439.040983:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574439.040988:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574439.040992:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574439.040994:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574439.040999:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574439.745030:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574439.745034:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574439.745036:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574440.064881:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574440.064890:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574440.064893:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574440.064901:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574440.064908:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574440.064943:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574440.064968:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574440.064973:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574440.064978:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574441.088946:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574441.088953:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574441.088956:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574441.088961:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574441.088977:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574441.088984:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574441.089006:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574441.089009:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574441.089013:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574441.793038:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574441.793041:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574441.793043:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574442.112901:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574442.112911:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574442.112917:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574442.112925:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574442.112954:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574442.112959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574442.112979:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574442.112998:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574442.113002:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574442.368883:0:265761:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309802221, at 4309802240 00000100:00000010:0.0:1788574442.368899:0:265761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574442.817036:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574442.817040:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574442.817042:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574443.136957:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574443.136965:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574443.136971:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574443.136982:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574443.136990:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574443.136999:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.137009:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574443.137016:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.137021:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.649127:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574443.649133:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574443.649137:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574443.649147:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516698112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.649163:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574443.649167:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574443.649172:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574443.649176:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574443.649180:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574443.649187:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516698112:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574443.649193:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574443.649197:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574443.649200:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574443.649205:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574443.649210:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574443.649216:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574443.649226:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574443.649231:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574443.649254:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574443.649262:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516698112, portal 25 00000100:00000001:0.0:1788574443.649264:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574443.649267:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788574443.649277:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516698112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.649290:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574443.649298:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574443.649302:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574443.649306:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516698112, offset 0 00000400:00000200:0.0:1788574443.649315:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574443.649322:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574443.649328:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574443.649333:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574443.649340:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574443.649347:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574443.649352:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574443.649358:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547106:547106:256:4294967295] 192.168.202.128@tcp LPNI seq info [547029:547029:8:4294967295] 00000400:00000200:0.0:1788574443.649367:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574443.649378:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574443.649388:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574443.649395:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574443.649404:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574443.649410:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574443.649433:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574443.649438:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574443.649444:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574443.649453:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574443.649462:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574443.649469:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574443.649478:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574443.649494:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516698112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.649522:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574443.649531:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516698112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.649554:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.649558:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574443.649568:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574443.649579:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574443.649586:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309807341 00000100:00000010:0.0:1788574443.649595:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574443.649607:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574443.649616:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309807341 00000100:00000010:0.0:1788574443.649625:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50f60. 00000100:00000001:0.0:1788574443.649632:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574443.649640:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574443.649648:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.649654:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574443.649664:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574443/real 1788574443] req@ffff8fc4ac821c00 x1875440516698112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574443.649683:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574443.649687:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574443.649691:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574443.649696:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574443.649698:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574443.649703:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574443.649709:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516698112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574443.649722:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516698112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.649732:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574443.649741:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.649746:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574443.649749:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574443.649756:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516698112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.649769:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574443.649777:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516698112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.649791:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574443.649795:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.649800:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574443.649803:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.649824:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574443.649826:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574443.649829:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574443.649833:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574443.649838:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574443.649842:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516698112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574443.649856:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516698112:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574443.649863:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.649867:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574443.649872:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516698112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.649883:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574443.649887:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574443.649890:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574443.649894:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574443.649898:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574443.649907:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574443.649910:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574443.649914:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788574443.649918:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574443.649922:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574443.649925:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788574443.649929:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574443.649933:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.649937:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.649945:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574443.649948:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.649965:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574443.649968:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574443.649971:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574443.649977:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516698240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.649987:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574443.649990:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574443.649992:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574443.649995:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574443.649999:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574443.650006:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516698240:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574443.650012:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574443.650016:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574443.650024:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574443.650028:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574443.650033:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574443.650036:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574443.650041:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574443.650045:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574443.650049:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574443.650053:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516698240, portal 10 00000100:00000001:0.0:1788574443.650054:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574443.650056:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788574443.650063:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516698240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650073:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574443.650075:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574443.650077:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574443.650079:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516698240, offset 0 00000400:00000200:0.0:1788574443.650081:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574443.650084:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574443.650087:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574443.650089:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574443.650091:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574443.650093:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574443.650095:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574443.650097:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547107:547107:256:4294967295] 192.168.202.128@tcp LPNI seq info [547030:547030:8:4294967295] 00000400:00000200:0.0:1788574443.650100:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574443.650105:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574443.650107:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574443.650109:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574443.650112:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574443.650115:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574443.650122:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574443.650124:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574443.650126:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574443.650128:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574443.650130:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574443.650132:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574443.650133:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574443.650136:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516698240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650142:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574443.650144:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516698240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650157:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.650158:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574443.650161:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574443.650164:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574443.650166:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574443.650168:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309803539 00000100:00000001:0.0:1788574443.650170:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309807341 00000100:00000001:0.0:1788574443.650171:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309803539 00000100:00000001:0.0:1788574443.650173:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309807341 00000100:00000001:0.0:1788574443.650174:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574443.650176:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.650177:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574443.650179:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574443/real 1788574443] req@ffff8fc4ac820a80 x1875440516698240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574443.650184:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574443.650186:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574443.650187:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574443.650188:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574443.650189:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574443.650191:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574443.650193:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516698240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574443.650198:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516698240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650202:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574443.650203:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650204:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574443.650205:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574443.650207:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516698240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650215:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574443.650218:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516698240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650223:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574443.650225:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650226:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574443.650227:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650228:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574443.650229:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574443.650231:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574443.650233:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574443.650234:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574443.650236:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516698240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574443.650241:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516698240:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574443.650244:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574443.650246:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516698368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650250:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574443.650251:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574443.650253:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574443.650254:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574443.650255:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574443.650257:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516698368:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574443.650259:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574443.650260:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574443.650264:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574443.650265:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574443.650267:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574443.650268:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574443.650271:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574443.650273:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574443.650275:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574443.650277:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516698368, portal 10 00000100:00000001:0.0:1788574443.650278:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574443.650279:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788574443.650283:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516698368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650288:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574443.650289:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574443.650291:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574443.650292:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516698368, offset 0 00000400:00000200:0.0:1788574443.650294:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574443.650298:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574443.650301:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574443.650304:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574443.650306:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574443.650308:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574443.650311:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574443.650315:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547108:547108:256:4294967295] 192.168.202.128@tcp LPNI seq info [547031:547031:8:4294967295] 00000400:00000200:0.0:1788574443.650319:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574443.650326:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574443.650331:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574443.650335:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574443.650339:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574443.650342:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574443.650354:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574443.650361:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574443.650364:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574443.650368:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574443.650373:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574443.650375:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574443.650377:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574443.650382:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516698368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650388:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574443.650390:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516698368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650394:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.650395:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574443.650398:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574443.650400:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574443.650402:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574443.650404:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309803546 00000100:00000001:0.0:1788574443.650406:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309807341 00000100:00000001:0.0:1788574443.650407:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309803546 00000100:00000001:0.0:1788574443.650409:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309807341 00000100:00000001:0.0:1788574443.650410:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574443.650412:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.650413:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574443.650415:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574443/real 1788574443] req@ffff8fc4ac823800 x1875440516698368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574443.650420:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574443.650421:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574443.650422:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574443.650424:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574443.650425:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574443.650431:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574443.650433:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516698368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574443.650439:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516698368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650443:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574443.650444:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650445:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574443.650446:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574443.650448:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516698368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650452:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574443.650455:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516698368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650459:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574443.650460:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650461:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574443.650462:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650463:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574443.650465:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574443.650466:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574443.650468:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574443.650469:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574443.650472:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516698368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574443.650477:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516698368:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574443.650480:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574443.650485:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516698496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650490:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574443.650491:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574443.650493:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574443.650494:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574443.650495:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574443.650497:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516698496:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574443.650499:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574443.650501:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574443.650502:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574443.650503:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574443.650505:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574443.650506:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574443.650509:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574443.650510:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574443.650512:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574443.650514:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516698496, portal 4 00000100:00000001:0.0:1788574443.650516:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574443.650517:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788574443.650521:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516698496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650525:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574443.650526:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574443.650528:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574443.650529:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516698496, offset 0 00000400:00000200:0.0:1788574443.650532:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574443.650534:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574443.650536:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574443.650538:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574443.650540:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574443.650542:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574443.650543:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574443.650548:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547109:547109:256:4294967295] 192.168.202.128@tcp LPNI seq info [547032:547032:8:4294967295] 00000400:00000200:0.0:1788574443.650551:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574443.650556:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574443.650559:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574443.650562:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574443.650564:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574443.650566:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574443.650577:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574443.650579:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574443.650581:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574443.650583:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574443.650585:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574443.650586:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574443.650588:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574443.650591:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516698496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650599:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574443.650603:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516698496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650609:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.650611:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574443.650614:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574443.650617:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574443.650620:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574443.650622:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309803686 00000100:00000001:0.0:1788574443.650625:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309807341 00000100:00000001:0.0:1788574443.650627:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309803686 00000100:00000001:0.0:1788574443.650630:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309807341 00000100:00000001:0.0:1788574443.650632:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574443.650640:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.650642:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574443.650647:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574443/real 1788574443] req@ffff8fc4ac820e00 x1875440516698496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574443.650656:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574443.650658:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574443.650660:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574443.650662:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574443.650664:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574443.650665:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574443.650668:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516698496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574443.650674:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516698496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650678:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574443.650679:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650680:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574443.650681:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574443.650683:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516698496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650688:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574443.650690:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516698496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650695:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574443.650696:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650697:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574443.650698:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650699:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574443.650701:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574443.650702:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574443.650708:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574443.650710:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574443.650713:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516698496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574443.650718:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516698496:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574443.650720:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650721:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574443.650723:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516698240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650727:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574443.650728:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574443.650730:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574443.650732:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574443.650733:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574443.650734:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574443.650735:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574443.650736:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574443.650738:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574443.650739:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574443.650740:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788574443.650742:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574443.650742:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650744:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574443.650746:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516698368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650750:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574443.650751:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574443.650752:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574443.650753:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574443.650754:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574443.650755:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574443.650756:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574443.650760:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788574443.650762:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574443.650763:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574443.650764:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788574443.650766:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574443.650767:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650768:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574443.650770:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516698496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650773:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574443.650774:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574443.650775:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574443.650776:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574443.650777:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574443.650778:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574443.650779:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574443.650779:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788574443.650781:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574443.650781:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574443.650782:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788574443.650783:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574443.650784:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650786:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.650788:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574443.650789:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.650791:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574443.650792:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574443.650793:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574443.650795:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516698624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650799:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574443.650800:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574443.650801:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574443.650802:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574443.650804:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574443.650806:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516698624:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574443.650826:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574443.650828:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574443.650829:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574443.650830:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574443.650832:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574443.650833:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574443.650836:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574443.650838:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574443.650840:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574443.650842:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516698624, portal 4 00000100:00000001:0.0:1788574443.650844:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574443.650845:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788574443.650849:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516698624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650853:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574443.650855:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574443.650856:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574443.650858:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516698624, offset 0 00000400:00000200:0.0:1788574443.650860:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574443.650863:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574443.650866:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574443.650869:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574443.650872:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574443.650875:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574443.650877:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574443.650880:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547110:547110:256:4294967295] 192.168.202.128@tcp LPNI seq info [547033:547033:8:4294967295] 00000400:00000200:0.0:1788574443.650885:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574443.650891:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574443.650896:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574443.650900:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574443.650904:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574443.650912:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574443.650925:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574443.650927:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574443.650930:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574443.650934:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574443.650938:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574443.650940:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574443.650943:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574443.650946:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516698624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650964:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574443.650967:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516698624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.650971:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.650972:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574443.650975:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574443.650977:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574443.650980:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574443.650981:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309803689 00000100:00000001:0.0:1788574443.650983:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309807341 00000100:00000001:0.0:1788574443.650984:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309803689 00000100:00000001:0.0:1788574443.650986:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309807341 00000100:00000001:0.0:1788574443.650988:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574443.650989:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.650990:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574443.650993:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574443/real 1788574443] req@ffff8fc4ac821880 x1875440516698624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574443.650997:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574443.650998:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574443.651004:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574443.651005:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574443.651006:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574443.651008:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574443.651011:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516698624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574443.651016:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516698624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.651021:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574443.651022:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.651023:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574443.651024:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574443.651026:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516698624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.651030:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574443.651032:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516698624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.651036:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574443.651038:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.651039:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574443.651039:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.651041:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574443.651042:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574443.651044:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574443.651046:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574443.651047:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574443.651050:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516698624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574443.651055:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516698624:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574443.651061:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.651062:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574443.651064:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516698624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574468 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574443.651069:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574443.651070:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574443.651071:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574443.651072:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574443.651073:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574443.651074:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574443.651075:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574443.651077:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862ac00. 02000000:00000001:0.0:1788574443.651078:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574443.651079:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574443.651080:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788574443.651082:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574443.651082:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.651083:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574443.651086:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574443.651087:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574443.651088:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574443.651089:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574444.160909:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574444.160917:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574444.160921:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574444.160928:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574444.160932:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574444.160936:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574444.161104:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574444.161106:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574444.161108:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574444.864975:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574444.864978:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574444.864980:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574445.185094:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574445.185101:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574445.185123:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574445.185128:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574445.185134:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574445.185137:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574445.185141:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574445.185142:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574445.185145:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574446.208933:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574446.208944:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574446.208949:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574446.208957:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574446.208964:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574446.208970:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574446.208997:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574446.209002:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574446.209005:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574446.913023:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574446.913032:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574446.913038:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574447.232923:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574447.232930:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574447.232935:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574447.232943:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574447.232950:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574447.232953:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574447.232964:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574447.232968:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574447.232971:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574447.233054:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574447.233057:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574447.233059:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574447.233061:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574447.233064:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574447.233068:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574447.233072:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574447.233074:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574447.233077:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574447.488922:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309807341, at 4309807360 00000100:00000010:0.0:1788574447.488954:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574447.937037:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574447.937046:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574447.937050:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574448.256952:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574448.256956:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574448.256960:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574448.256963:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574448.256968:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574448.256973:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.256981:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574448.256984:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.256987:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.769151:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574448.769157:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574448.769161:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574448.769171:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516698752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.769191:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574448.769195:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574448.769199:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574448.769204:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574448.769208:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574448.769214:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516698752:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574448.769221:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574448.769225:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574448.769229:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574448.769232:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574448.769238:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574448.769241:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574448.769249:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574448.769254:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574448.769261:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574448.769270:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516698752, portal 25 00000100:00000001:0.0:1788574448.769274:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574448.769284:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788574448.769292:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516698752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.769315:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574448.769320:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574448.769324:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574448.769328:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516698752, offset 0 00000400:00000200:0.0:1788574448.769338:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574448.769348:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574448.769359:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574448.769366:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574448.769373:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574448.769380:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574448.769385:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574448.769394:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547111:547111:256:4294967295] 192.168.202.128@tcp LPNI seq info [547034:547034:8:4294967295] 00000400:00000200:0.0:1788574448.769403:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574448.769422:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574448.769434:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574448.769439:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574448.769449:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574448.769455:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574448.769475:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574448.769478:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574448.769483:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574448.769489:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574448.769494:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574448.769497:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574448.769503:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574448.769510:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516698752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.769527:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574448.769543:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516698752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.769579:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.769584:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574448.769592:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574448.769599:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574448.769603:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309812461 00000100:00000010:0.0:1788574448.769610:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574448.769619:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574448.769625:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309812461 00000100:00000010:0.0:1788574448.769631:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50420. 00000100:00000001:0.0:1788574448.769637:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574448.769643:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574448.769647:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.769652:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574448.769660:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574448/real 1788574448] req@ffff8fc4901a4a80 x1875440516698752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574448.769682:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574448.769687:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574448.769694:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574448.769702:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574448.769707:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574448.769715:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574448.769724:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516698752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574448.769748:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516698752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.769762:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574448.769765:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.769770:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574448.769775:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574448.769785:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516698752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.769827:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574448.769834:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516698752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.769850:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574448.769855:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.769859:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574448.769863:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.769868:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574448.769872:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574448.769877:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574448.769885:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574448.769890:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574448.769898:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516698752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574448.769918:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516698752:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574448.769926:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.769930:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574448.769935:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516698752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.769963:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574448.769968:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574448.769973:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574448.769980:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574448.769984:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574448.769989:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574448.769995:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574448.770001:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788574448.770020:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574448.770031:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574448.770034:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788574448.770040:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574448.770043:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770047:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770055:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574448.770058:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.770060:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574448.770063:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574448.770066:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574448.770073:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516698880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770088:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574448.770091:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574448.770093:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574448.770096:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574448.770101:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574448.770111:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516698880:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574448.770119:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574448.770122:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574448.770124:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574448.770127:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574448.770131:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574448.770134:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574448.770140:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574448.770143:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574448.770149:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574448.770152:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516698880, portal 10 00000100:00000001:0.0:1788574448.770154:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574448.770156:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788574448.770163:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516698880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770168:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574448.770170:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574448.770176:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574448.770178:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516698880, offset 0 00000400:00000200:0.0:1788574448.770182:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574448.770185:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574448.770187:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574448.770190:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574448.770192:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574448.770194:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574448.770196:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574448.770198:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547112:547112:256:4294967295] 192.168.202.128@tcp LPNI seq info [547035:547035:8:4294967295] 00000400:00000200:0.0:1788574448.770200:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574448.770206:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574448.770211:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574448.770215:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574448.770219:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574448.770222:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574448.770232:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574448.770234:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574448.770237:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574448.770241:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574448.770244:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574448.770247:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574448.770250:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574448.770254:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516698880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770262:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574448.770266:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516698880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770271:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.770272:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574448.770280:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574448.770283:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574448.770286:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574448.770288:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309808659 00000100:00000001:0.0:1788574448.770290:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309812461 00000100:00000001:0.0:1788574448.770291:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309808659 00000100:00000001:0.0:1788574448.770293:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309812461 00000100:00000001:0.0:1788574448.770295:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574448.770296:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.770297:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574448.770300:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574448/real 1788574448] req@ffff8fc4901a7100 x1875440516698880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574448.770305:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574448.770306:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574448.770308:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574448.770309:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574448.770311:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574448.770312:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574448.770314:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516698880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574448.770319:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516698880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770323:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574448.770324:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770325:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574448.770326:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574448.770328:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516698880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770332:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574448.770335:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516698880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770343:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574448.770345:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770346:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574448.770348:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770350:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574448.770351:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574448.770354:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574448.770358:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574448.770360:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574448.770365:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516698880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574448.770374:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516698880:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574448.770378:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574448.770382:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516699008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770389:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574448.770390:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574448.770392:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574448.770394:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574448.770397:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574448.770400:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699008:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574448.770403:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574448.770405:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574448.770407:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574448.770409:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574448.770413:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574448.770415:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574448.770424:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574448.770427:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574448.770431:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574448.770434:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516699008, portal 10 00000100:00000001:0.0:1788574448.770436:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574448.770437:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788574448.770442:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516699008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770447:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574448.770448:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574448.770450:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574448.770452:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516699008, offset 0 00000400:00000200:0.0:1788574448.770454:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574448.770457:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574448.770460:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574448.770462:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574448.770464:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574448.770465:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574448.770467:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574448.770469:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547113:547113:256:4294967295] 192.168.202.128@tcp LPNI seq info [547036:547036:8:4294967295] 00000400:00000200:0.0:1788574448.770472:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574448.770477:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574448.770480:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574448.770482:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574448.770484:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574448.770486:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574448.770494:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574448.770495:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574448.770497:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574448.770500:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574448.770503:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574448.770510:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574448.770513:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574448.770517:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516699008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770526:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574448.770529:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516699008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770536:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.770538:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574448.770542:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574448.770545:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574448.770549:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574448.770551:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309808666 00000100:00000001:0.0:1788574448.770553:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309812461 00000100:00000001:0.0:1788574448.770555:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309808666 00000100:00000001:0.0:1788574448.770557:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309812461 00000100:00000001:0.0:1788574448.770559:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574448.770560:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.770561:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574448.770564:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574448/real 1788574448] req@ffff8fc4901a5180 x1875440516699008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574448.770569:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574448.770570:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574448.770571:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574448.770573:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574448.770574:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574448.770576:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574448.770578:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516699008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574448.770584:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516699008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770592:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574448.770594:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770595:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574448.770596:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574448.770599:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516699008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770603:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574448.770605:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516699008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770609:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574448.770611:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770612:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574448.770613:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770615:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574448.770616:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574448.770618:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574448.770620:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574448.770621:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574448.770624:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516699008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574448.770629:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699008:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574448.770632:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574448.770634:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516699136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770638:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574448.770639:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574448.770645:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574448.770646:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574448.770648:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574448.770650:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699136:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574448.770652:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574448.770653:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574448.770654:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574448.770656:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574448.770658:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574448.770659:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574448.770662:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574448.770663:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574448.770666:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574448.770668:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516699136, portal 4 00000100:00000001:0.0:1788574448.770669:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574448.770670:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788574448.770674:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516699136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770678:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574448.770680:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574448.770681:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574448.770683:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516699136, offset 0 00000400:00000200:0.0:1788574448.770685:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574448.770688:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574448.770690:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574448.770692:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574448.770694:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574448.770696:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574448.770698:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574448.770700:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547114:547114:256:4294967295] 192.168.202.128@tcp LPNI seq info [547037:547037:8:4294967295] 00000400:00000200:0.0:1788574448.770703:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574448.770707:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574448.770713:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574448.770715:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574448.770718:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574448.770720:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574448.770727:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574448.770728:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574448.770730:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574448.770733:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574448.770735:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574448.770736:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574448.770738:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574448.770740:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516699136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770745:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574448.770747:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516699136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770751:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.770752:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574448.770755:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574448.770757:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574448.770759:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574448.770760:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309808806 00000100:00000001:0.0:1788574448.770762:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309812461 00000100:00000001:0.0:1788574448.770763:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309808806 00000100:00000001:0.0:1788574448.770765:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309812461 00000100:00000001:0.0:1788574448.770766:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574448.770767:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.770768:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574448.770771:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574448/real 1788574448] req@ffff8fc4901a5c00 x1875440516699136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574448.770779:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574448.770780:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574448.770781:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574448.770783:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574448.770784:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574448.770785:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574448.770787:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516699136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574448.770793:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516699136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770797:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574448.770797:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770799:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574448.770799:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574448.770801:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516699136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770805:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574448.770824:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516699136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770829:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574448.770831:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770832:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574448.770833:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770834:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574448.770835:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574448.770837:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574448.770839:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574448.770840:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574448.770847:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516699136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574448.770852:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699136:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574448.770855:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770856:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574448.770858:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516698880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770862:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574448.770863:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574448.770865:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574448.770867:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574448.770868:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574448.770869:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574448.770870:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574448.770872:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788574448.770874:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574448.770875:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574448.770876:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788574448.770879:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574448.770880:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770881:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574448.770883:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516699008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770887:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574448.770888:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574448.770889:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574448.770890:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574448.770891:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574448.770892:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574448.770893:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574448.770894:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788574448.770896:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574448.770901:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574448.770903:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788574448.770907:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574448.770908:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770910:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574448.770914:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516699136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770920:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574448.770922:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574448.770924:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574448.770926:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574448.770927:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574448.770928:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574448.770930:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574448.770932:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788574448.770933:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574448.770934:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574448.770936:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788574448.770938:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574448.770939:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770941:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.770945:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574448.770946:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.770948:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574448.770950:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574448.770951:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574448.770955:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516699264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.770962:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574448.770963:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574448.770965:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574448.770967:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574448.770969:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574448.770971:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699264:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574448.770974:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574448.770982:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574448.770983:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574448.770985:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574448.770988:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574448.770990:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574448.770994:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574448.770996:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574448.770998:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574448.771000:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516699264, portal 4 00000100:00000001:0.0:1788574448.771002:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574448.771003:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788574448.771020:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516699264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.771026:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574448.771027:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574448.771029:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574448.771031:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516699264, offset 0 00000400:00000200:0.0:1788574448.771034:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574448.771037:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574448.771039:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574448.771041:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574448.771043:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574448.771045:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574448.771047:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574448.771049:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547115:547115:256:4294967295] 192.168.202.128@tcp LPNI seq info [547038:547038:8:4294967295] 00000400:00000200:0.0:1788574448.771051:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574448.771056:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574448.771059:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574448.771061:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574448.771064:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574448.771066:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574448.771073:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574448.771078:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574448.771081:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574448.771083:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574448.771086:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574448.771087:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574448.771089:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574448.771091:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516699264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.771097:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574448.771100:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516699264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574473 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.771106:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.771108:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574448.771111:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574448.771114:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574448.771117:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574448.771120:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309808809 00000100:00000001:0.0:1788574448.771122:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309812461 00000100:00000001:0.0:1788574448.771124:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309808809 00000100:00000001:0.0:1788574448.771126:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309812461 00000100:00000001:0.0:1788574448.771129:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574448.771131:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.771134:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574448.771138:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574448/real 1788574448] req@ffff8fc4901a7800 x1875440516699264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574448.771148:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574448.771149:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574448.771151:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574448.771154:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574448.771155:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574448.771163:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574448.771165:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516699264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574448.771171:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516699264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.771178:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574448.771180:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.771182:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574448.771183:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574448.771187:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516699264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.771195:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574448.771199:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516699264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.771207:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574448.771209:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.771211:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574448.771212:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.771214:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574448.771216:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574448.771219:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574448.771222:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574448.771225:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574448.771229:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516699264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574448.771238:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699264:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574448.771243:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.771245:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574448.771254:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516699264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574473 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574448.771261:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574448.771262:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574448.771264:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574448.771266:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574448.771267:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574448.771268:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574448.771269:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574448.771270:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788574448.771272:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574448.771273:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574448.771274:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788574448.771276:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574448.771277:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.771278:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574448.771281:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574448.771282:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574448.771283:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574448.771285:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574449.280942:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574449.280948:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574449.280952:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574449.280959:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574449.280965:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574449.280971:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574449.280978:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574449.280982:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574449.280986:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574449.985062:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574449.985065:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574449.985068:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574450.304985:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574450.305001:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574450.305012:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574450.305038:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574450.305045:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574450.305083:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574450.305100:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574450.305105:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574450.305117:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574451.328965:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574451.328979:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574451.328983:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574451.328987:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574451.328996:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574451.329005:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574451.329018:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574451.329022:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574451.329025:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574451.329132:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574451.329136:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574451.329140:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574451.329149:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574451.329155:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574451.329160:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574451.329174:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574451.329179:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574451.329183:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574452.033028:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574452.033032:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574452.033034:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574452.352948:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574452.352959:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574452.352962:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574452.352970:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574452.352981:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574452.352988:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574452.352996:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574452.353000:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574452.353006:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574452.353044:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574452.353064:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574452.353070:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574452.353075:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574452.353099:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574452.353101:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574452.353104:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574452.353105:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574452.353107:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574452.608910:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309812461, at 4309812480 00000100:00000010:0.0:1788574452.608927:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574453.056969:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574453.056973:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574453.056975:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574453.376924:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574453.376930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574453.376934:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574453.376942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574453.376949:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574453.376954:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.376971:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574453.376975:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.376978:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.889247:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574453.889253:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574453.889257:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574453.889266:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516699392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.889286:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574453.889291:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574453.889298:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574453.889303:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574453.889310:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574453.889318:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516699392:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574453.889328:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574453.889334:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574453.889338:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574453.889343:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574453.889349:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574453.889360:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574453.889368:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574453.889372:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574453.889380:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574453.889388:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516699392, portal 25 00000100:00000001:0.0:1788574453.889393:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574453.889396:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788574453.889402:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516699392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.889416:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574453.889421:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574453.889425:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574453.889431:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516699392, offset 0 00000400:00000200:0.0:1788574453.889439:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574453.889448:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574453.889456:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574453.889461:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574453.889466:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574453.889472:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574453.889477:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574453.889485:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547116:547116:256:4294967295] 192.168.202.128@tcp LPNI seq info [547039:547039:8:4294967295] 00000400:00000200:0.0:1788574453.889498:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574453.889519:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574453.889534:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574453.889543:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574453.889554:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574453.889562:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574453.889588:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574453.889593:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574453.889600:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574453.889609:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574453.889623:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574453.889626:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574453.889634:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574453.889641:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516699392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.889655:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574453.889661:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516699392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.889672:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.889676:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574453.889681:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574453.889685:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574453.889688:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309817581 00000100:00000010:0.0:1788574453.889693:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574453.889701:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574453.889708:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309817581 00000100:00000010:0.0:1788574453.889713:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50a20. 00000100:00000001:0.0:1788574453.889717:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574453.889721:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574453.889724:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.889729:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574453.889734:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574453/real 1788574453] req@ffff8fc4901a5880 x1875440516699392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574453.889743:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574453.889744:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574453.889746:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574453.889750:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574453.889753:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574453.889757:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574453.889761:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516699392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574453.889772:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516699392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.889784:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574453.889786:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.889791:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574453.889793:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574453.889798:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516699392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.889825:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574453.889830:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516699392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.889836:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574453.889841:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.889844:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574453.889846:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.889850:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574453.889852:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574453.889858:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574453.889862:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574453.889865:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574453.889872:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516699392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574453.889884:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516699392:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574453.889903:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.889912:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574453.889917:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516699392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.889930:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574453.889933:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574453.889942:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574453.889947:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574453.889951:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574453.889954:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574453.889958:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574453.889962:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788574453.889967:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574453.889970:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574453.889973:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788574453.889979:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574453.889982:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.889985:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.889994:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574453.889998:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.890000:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574453.890004:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574453.890007:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574453.890013:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516699520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890028:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574453.890032:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574453.890035:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574453.890037:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574453.890042:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574453.890050:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699520:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574453.890059:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574453.890076:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574453.890079:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574453.890084:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574453.890089:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574453.890095:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574453.890102:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574453.890106:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574453.890118:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574453.890124:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516699520, portal 10 00000100:00000001:0.0:1788574453.890132:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574453.890135:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788574453.890143:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516699520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890153:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574453.890158:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574453.890161:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574453.890163:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516699520, offset 0 00000400:00000200:0.0:1788574453.890170:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574453.890176:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574453.890183:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574453.890186:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574453.890191:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574453.890195:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574453.890200:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574453.890206:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547117:547117:256:4294967295] 192.168.202.128@tcp LPNI seq info [547040:547040:8:4294967295] 00000400:00000200:0.0:1788574453.890213:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574453.890224:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574453.890231:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574453.890236:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574453.890244:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574453.890249:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574453.890265:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574453.890269:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574453.890276:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574453.890280:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574453.890287:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574453.890292:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574453.890297:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574453.890306:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516699520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890343:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574453.890350:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516699520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890361:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.890364:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574453.890370:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574453.890375:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574453.890382:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574453.890386:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309813779 00000100:00000001:0.0:1788574453.890390:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309817581 00000100:00000001:0.0:1788574453.890393:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309813779 00000100:00000001:0.0:1788574453.890398:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309817581 00000100:00000001:0.0:1788574453.890402:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574453.890405:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.890409:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574453.890414:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574453/real 1788574453] req@ffff8fc4901a4700 x1875440516699520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574453.890425:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574453.890428:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574453.890431:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574453.890434:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574453.890438:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574453.890441:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574453.890446:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516699520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574453.890459:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516699520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890469:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574453.890475:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.890478:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574453.890479:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574453.890481:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516699520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890491:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574453.890497:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516699520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890507:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574453.890510:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.890514:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574453.890517:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.890520:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574453.890524:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574453.890526:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574453.890531:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574453.890533:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574453.890540:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516699520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574453.890553:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699520:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574453.890558:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574453.890563:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516699648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890581:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574453.890584:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574453.890587:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574453.890591:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574453.890593:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574453.890605:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699648:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574453.890613:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574453.890615:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574453.890619:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574453.890621:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574453.890626:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574453.890629:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574453.890637:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574453.890639:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574453.890641:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574453.890644:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516699648, portal 10 00000100:00000001:0.0:1788574453.890645:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574453.890646:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788574453.890652:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516699648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890657:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574453.890660:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574453.890663:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574453.890665:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516699648, offset 0 00000400:00000200:0.0:1788574453.890669:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574453.890673:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574453.890677:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574453.890681:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574453.890684:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574453.890686:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574453.890689:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574453.890692:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547118:547118:256:4294967295] 192.168.202.128@tcp LPNI seq info [547041:547041:8:4294967295] 00000400:00000200:0.0:1788574453.890696:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574453.890705:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574453.890709:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574453.890712:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574453.890719:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574453.890721:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574453.890729:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574453.890731:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574453.890733:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574453.890737:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574453.890739:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574453.890741:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574453.890743:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574453.890745:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516699648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890751:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574453.890753:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516699648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890757:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.890758:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574453.890761:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574453.890763:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574453.890765:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574453.890766:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309813786 00000100:00000001:0.0:1788574453.890768:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309817581 00000100:00000001:0.0:1788574453.890769:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309813786 00000100:00000001:0.0:1788574453.890771:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309817581 00000100:00000001:0.0:1788574453.890772:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574453.890774:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.890775:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574453.890777:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574453/real 1788574453] req@ffff8fc4901a7480 x1875440516699648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574453.890782:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574453.890787:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574453.890788:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574453.890790:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574453.890791:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574453.890793:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574453.890795:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516699648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574453.890801:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516699648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890804:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574453.890805:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.890822:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574453.890824:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574453.890827:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516699648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890831:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574453.890833:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516699648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890838:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574453.890840:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.890841:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574453.890841:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.890842:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574453.890844:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574453.890846:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574453.890848:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574453.890849:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574453.890851:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516699648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574453.890865:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699648:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574453.890867:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.890869:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574453.890871:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516699520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890875:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574453.890876:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574453.890878:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574453.890880:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574453.890881:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574453.890882:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574453.890883:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574453.890885:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788574453.890887:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574453.890888:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574453.890889:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788574453.890892:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574453.890893:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.890894:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574453.890896:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516699648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890901:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574453.890901:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574453.890902:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574453.890903:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574453.890904:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574453.890905:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574453.890906:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574453.890907:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788574453.890908:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574453.890908:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574453.890909:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788574453.890914:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574453.890915:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.890917:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.890919:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574453.890920:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.890922:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574453.890923:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574453.890924:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574453.890926:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516699776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890931:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574453.890932:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574453.890934:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574453.890935:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574453.890936:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574453.890938:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699776:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574453.890940:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574453.890941:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574453.890942:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574453.890943:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574453.890945:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574453.890946:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574453.890949:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574453.890950:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574453.890953:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574453.890955:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516699776, portal 4 00000100:00000001:0.0:1788574453.890957:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574453.890957:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788574453.890961:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516699776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.890966:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574453.890968:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574453.890969:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574453.890971:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516699776, offset 0 00000400:00000200:0.0:1788574453.890977:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574453.890979:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574453.890982:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574453.890984:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574453.890986:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574453.890987:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574453.890989:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574453.890991:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547119:547119:256:4294967295] 192.168.202.128@tcp LPNI seq info [547042:547042:8:4294967295] 00000400:00000200:0.0:1788574453.890994:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574453.890998:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574453.891001:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574453.891003:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574453.891006:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574453.891008:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574453.891015:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574453.891016:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574453.891018:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574453.891020:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574453.891022:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574453.891023:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574453.891025:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574453.891027:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516699776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891034:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574453.891037:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516699776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891044:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.891046:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574453.891049:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574453.891052:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574453.891056:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574453.891063:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309813927 00000100:00000001:0.0:1788574453.891078:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309817581 00000100:00000001:0.0:1788574453.891081:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309813927 00000100:00000001:0.0:1788574453.891083:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309817581 00000100:00000001:0.0:1788574453.891085:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574453.891088:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.891090:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574453.891095:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574453/real 1788574453] req@ffff8fc4901a6300 x1875440516699776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574453.891104:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574453.891105:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574453.891106:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574453.891108:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574453.891109:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574453.891111:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574453.891114:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516699776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574453.891119:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516699776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891124:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574453.891124:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.891126:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574453.891126:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574453.891129:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516699776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891132:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574453.891135:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516699776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891139:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574453.891146:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.891147:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574453.891148:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.891149:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574453.891151:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574453.891153:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574453.891155:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574453.891156:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574453.891159:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516699776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574453.891164:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699776:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574453.891166:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574453.891168:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516699904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891172:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574453.891173:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574453.891174:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574453.891176:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574453.891177:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574453.891179:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699904:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574453.891180:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574453.891182:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574453.891183:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574453.891184:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574453.891186:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574453.891188:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574453.891190:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574453.891192:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574453.891194:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574453.891196:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516699904, portal 4 00000100:00000001:0.0:1788574453.891200:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574453.891201:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788574453.891205:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516699904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891210:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574453.891211:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574453.891213:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574453.891214:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516699904, offset 0 00000400:00000200:0.0:1788574453.891217:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574453.891220:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574453.891222:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574453.891224:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574453.891226:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574453.891228:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574453.891229:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574453.891231:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547120:547120:256:4294967295] 192.168.202.128@tcp LPNI seq info [547043:547043:8:4294967295] 00000400:00000200:0.0:1788574453.891234:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574453.891239:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574453.891241:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574453.891243:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574453.891246:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574453.891248:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574453.891255:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574453.891256:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574453.891258:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574453.891260:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574453.891262:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574453.891263:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574453.891265:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574453.891267:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516699904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891285:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574453.891289:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516699904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574478 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891295:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.891297:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574453.891302:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574453.891305:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574453.891308:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574453.891311:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309813929 00000100:00000001:0.0:1788574453.891313:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309817581 00000100:00000001:0.0:1788574453.891316:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309813929 00000100:00000001:0.0:1788574453.891318:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309817581 00000100:00000001:0.0:1788574453.891320:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574453.891323:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.891325:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574453.891329:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574453/real 1788574453] req@ffff8fc4901a4e00 x1875440516699904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574453.891338:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574453.891340:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574453.891341:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574453.891344:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574453.891345:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574453.891347:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574453.891350:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516699904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574453.891355:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516699904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891359:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574453.891365:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.891367:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574453.891368:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574453.891370:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516699904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891375:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574453.891378:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516699904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891382:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574453.891384:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.891385:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574453.891386:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.891387:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574453.891388:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574453.891390:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574453.891391:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574453.891393:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574453.891395:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516699904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574453.891400:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516699904:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574453.891402:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.891403:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574453.891405:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516699776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891410:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574453.891411:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574453.891412:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574453.891414:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574453.891415:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574453.891418:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574453.891419:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574453.891421:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788574453.891423:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574453.891424:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574453.891425:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788574453.891428:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574453.891429:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.891431:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574453.891433:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516699904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574478 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574453.891437:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574453.891438:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574453.891439:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574453.891440:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574453.891441:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574453.891441:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574453.891442:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574453.891443:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788574453.891444:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574453.891445:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574453.891446:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788574453.891448:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574453.891449:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.891450:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574453.891452:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574453.891453:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574453.891454:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574453.891455:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574454.400932:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574454.400937:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574454.400941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574454.400945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574454.400949:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574454.400953:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574454.400960:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574454.400963:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574454.400981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574455.104997:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574455.105000:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574455.105003:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574455.424916:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574455.424923:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574455.424928:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574455.424935:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574455.424941:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574455.424945:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574455.424953:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574455.424956:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574455.424962:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574456.448900:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574456.448906:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574456.448910:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574456.448914:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574456.448919:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574456.448926:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574456.448943:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574456.448946:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574456.448949:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574456.448973:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574456.448976:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574456.448978:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574456.448980:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574456.448983:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574456.448985:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574456.448987:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574456.448988:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574456.448990:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574456.960934:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788574456.960942:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788574456.960947:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788574456.961152:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574456.961169:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574456.961176:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574456.961189:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788574457.152991:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574457.152995:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574457.152997:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574457.728915:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309817581, at 4309817600 00000100:00000010:0.0:1788574457.728935:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574458.177030:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574458.177034:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574458.177038:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574458.496946:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574458.496960:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574458.496964:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574458.496976:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574458.496991:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574458.496998:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574458.497003:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574458.497006:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574458.497011:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.009109:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574459.009115:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574459.009120:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574459.009151:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516700032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.009172:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574459.009176:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574459.009180:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574459.009183:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574459.009186:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574459.009192:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516700032:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574459.009197:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574459.009203:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574459.009208:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574459.009215:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574459.009224:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574459.009231:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574459.009252:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574459.009258:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574459.009266:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574459.009273:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516700032, portal 25 00000100:00000001:0.0:1788574459.009278:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574459.009282:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788574459.009290:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516700032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.009302:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574459.009307:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574459.009310:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574459.009314:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516700032, offset 0 00000400:00000200:0.0:1788574459.009321:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574459.009333:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574459.009342:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574459.009349:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574459.009356:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574459.009362:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574459.009366:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574459.009373:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547121:547121:256:4294967295] 192.168.202.128@tcp LPNI seq info [547044:547044:8:4294967295] 00000400:00000200:0.0:1788574459.009385:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574459.009402:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574459.009413:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574459.009418:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574459.009423:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574459.009429:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574459.009453:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574459.009458:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574459.009463:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574459.009469:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574459.009475:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574459.009486:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574459.009494:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574459.009502:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516700032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.009524:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574459.009532:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516700032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.009553:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.009558:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574459.009566:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574459.009575:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574459.009582:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309822701 00000100:00000010:0.0:1788574459.009589:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574459.009599:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574459.009609:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309822701 00000100:00000010:0.0:1788574459.009617:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b506c0. 00000100:00000001:0.0:1788574459.009624:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574459.009632:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574459.009639:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.009646:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574459.009657:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574459/real 1788574459] req@ffff8fc4901a4e00 x1875440516700032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574459.009677:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574459.009681:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574459.009685:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574459.009690:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574459.009694:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574459.009698:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574459.009706:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516700032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574459.009731:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516700032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.009764:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574459.009769:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.009774:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574459.009777:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574459.009785:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516700032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.009801:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574459.009826:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516700032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.009831:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574459.009835:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.009838:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574459.009840:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.009844:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574459.009847:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574459.009854:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574459.009865:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574459.009870:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574459.009881:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516700032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574459.009902:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516700032:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574459.009909:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.009912:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574459.009917:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516700032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.009928:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574459.009931:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574459.009941:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574459.009945:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574459.009949:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574459.009952:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574459.009956:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574459.009961:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788574459.009968:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574459.009974:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574459.009979:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788574459.009990:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574459.009995:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010000:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010010:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574459.010015:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.010022:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574459.010026:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574459.010029:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574459.010035:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516700160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010050:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574459.010052:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574459.010055:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574459.010057:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574459.010059:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574459.010062:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516700160:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574459.010064:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574459.010067:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574459.010068:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574459.010070:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574459.010073:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574459.010075:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574459.010079:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574459.010082:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574459.010086:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574459.010089:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516700160, portal 10 00000100:00000001:0.0:1788574459.010094:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574459.010096:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788574459.010102:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516700160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010106:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574459.010108:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574459.010110:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574459.010112:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516700160, offset 0 00000400:00000200:0.0:1788574459.010114:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574459.010117:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574459.010120:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574459.010135:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574459.010137:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574459.010139:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574459.010141:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574459.010143:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547122:547122:256:4294967295] 192.168.202.128@tcp LPNI seq info [547045:547045:8:4294967295] 00000400:00000200:0.0:1788574459.010146:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574459.010151:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574459.010154:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574459.010157:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574459.010160:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574459.010162:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574459.010170:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574459.010171:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574459.010173:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574459.010176:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574459.010178:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574459.010179:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574459.010181:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574459.010184:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516700160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010198:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574459.010201:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516700160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010209:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.010210:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574459.010215:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574459.010219:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574459.010222:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574459.010225:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309818899 00000100:00000001:0.0:1788574459.010228:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309822701 00000100:00000001:0.0:1788574459.010230:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309818899 00000100:00000001:0.0:1788574459.010233:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309822701 00000100:00000001:0.0:1788574459.010235:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574459.010238:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.010240:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574459.010244:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574459/real 1788574459] req@ffff8fc4901a6300 x1875440516700160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574459.010254:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574459.010255:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574459.010257:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574459.010258:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574459.010259:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574459.010261:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574459.010264:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516700160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574459.010269:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516700160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010274:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574459.010279:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010281:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574459.010282:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574459.010284:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516700160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010289:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574459.010292:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516700160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010296:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574459.010298:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010299:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574459.010300:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010301:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574459.010302:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574459.010304:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574459.010306:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574459.010307:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574459.010309:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516700160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574459.010314:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516700160:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574459.010317:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574459.010319:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516700288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010324:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574459.010325:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574459.010326:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574459.010327:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574459.010328:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574459.010333:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516700288:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574459.010335:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574459.010337:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574459.010338:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574459.010339:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574459.010341:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574459.010343:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574459.010345:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574459.010347:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574459.010349:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574459.010351:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516700288, portal 10 00000100:00000001:0.0:1788574459.010353:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574459.010354:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788574459.010358:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516700288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010362:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574459.010363:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574459.010365:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574459.010366:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516700288, offset 0 00000400:00000200:0.0:1788574459.010369:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574459.010371:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574459.010374:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574459.010376:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574459.010378:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574459.010379:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574459.010381:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574459.010383:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547123:547123:256:4294967295] 192.168.202.128@tcp LPNI seq info [547046:547046:8:4294967295] 00000400:00000200:0.0:1788574459.010386:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574459.010390:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574459.010393:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574459.010395:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574459.010402:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574459.010404:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574459.010411:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574459.010412:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574459.010414:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574459.010417:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574459.010419:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574459.010420:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574459.010422:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574459.010424:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516700288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010429:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574459.010431:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516700288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010436:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.010437:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574459.010439:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574459.010441:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574459.010444:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574459.010445:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309818906 00000100:00000001:0.0:1788574459.010447:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309822701 00000100:00000001:0.0:1788574459.010448:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309818906 00000100:00000001:0.0:1788574459.010449:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309822701 00000100:00000001:0.0:1788574459.010451:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574459.010452:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.010453:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574459.010455:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574459/real 1788574459] req@ffff8fc4901a7480 x1875440516700288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574459.010460:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574459.010464:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574459.010465:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574459.010467:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574459.010468:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574459.010470:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574459.010472:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516700288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574459.010477:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516700288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010481:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574459.010482:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010483:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574459.010484:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574459.010486:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516700288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010490:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574459.010493:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516700288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010497:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574459.010498:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010500:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574459.010500:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010502:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574459.010504:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574459.010505:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574459.010507:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574459.010508:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574459.010510:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516700288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574459.010522:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516700288:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574459.010525:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574459.010527:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516700416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010531:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574459.010532:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574459.010533:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574459.010534:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574459.010536:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574459.010537:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516700416:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574459.010539:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574459.010540:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574459.010542:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574459.010543:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574459.010545:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574459.010546:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574459.010549:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574459.010550:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574459.010553:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574459.010555:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516700416, portal 4 00000100:00000001:0.0:1788574459.010556:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574459.010557:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788574459.010561:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516700416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010565:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574459.010566:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574459.010568:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574459.010569:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516700416, offset 0 00000400:00000200:0.0:1788574459.010571:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574459.010577:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574459.010580:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574459.010582:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574459.010583:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574459.010585:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574459.010587:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574459.010589:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547124:547124:256:4294967295] 192.168.202.128@tcp LPNI seq info [547047:547047:8:4294967295] 00000400:00000200:0.0:1788574459.010591:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574459.010596:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574459.010599:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574459.010600:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574459.010603:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574459.010604:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574459.010611:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574459.010612:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574459.010614:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574459.010616:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574459.010618:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574459.010619:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574459.010621:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574459.010623:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516700416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010628:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574459.010630:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516700416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010634:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.010635:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574459.010637:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574459.010639:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574459.010641:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574459.010646:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309819046 00000100:00000001:0.0:1788574459.010647:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309822701 00000100:00000001:0.0:1788574459.010649:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309819046 00000100:00000001:0.0:1788574459.010650:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309822701 00000100:00000001:0.0:1788574459.010652:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574459.010653:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.010654:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574459.010657:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574459/real 1788574459] req@ffff8fc4901a4700 x1875440516700416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574459.010662:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574459.010663:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574459.010664:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574459.010665:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574459.010666:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574459.010667:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574459.010669:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516700416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574459.010675:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516700416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010679:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574459.010679:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010681:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574459.010681:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574459.010683:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516700416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010687:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574459.010690:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516700416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010694:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574459.010698:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010699:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574459.010701:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010702:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574459.010704:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574459.010707:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574459.010710:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574459.010713:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574459.010717:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516700416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574459.010726:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516700416:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574459.010729:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010732:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574459.010735:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516700160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010742:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574459.010744:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574459.010747:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574459.010749:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574459.010751:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574459.010752:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574459.010754:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574459.010755:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788574459.010757:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574459.010758:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574459.010759:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788574459.010763:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574459.010764:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010765:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574459.010768:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516700288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010781:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574459.010782:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574459.010783:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574459.010784:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574459.010785:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574459.010786:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574459.010787:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574459.010788:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788574459.010789:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574459.010790:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574459.010791:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788574459.010793:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574459.010794:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010795:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574459.010797:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516700416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010801:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574459.010802:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574459.010803:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574459.010805:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574459.010806:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574459.010822:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574459.010824:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574459.010825:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788574459.010826:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574459.010827:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574459.010828:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788574459.010829:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574459.010830:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010831:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.010834:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574459.010835:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.010837:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574459.010838:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574459.010839:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574459.010846:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516700544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010853:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574459.010855:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574459.010857:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574459.010859:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574459.010861:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574459.010863:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516700544:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574459.010867:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574459.010869:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574459.010870:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574459.010872:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574459.010875:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574459.010877:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574459.010881:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574459.010884:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574459.010888:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574459.010892:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516700544, portal 4 00000100:00000001:0.0:1788574459.010894:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574459.010895:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788574459.010901:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516700544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010906:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574459.010908:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574459.010909:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574459.010911:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516700544, offset 0 00000400:00000200:0.0:1788574459.010914:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574459.010916:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574459.010919:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574459.010921:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574459.010923:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574459.010925:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574459.010926:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574459.010932:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547125:547125:256:4294967295] 192.168.202.128@tcp LPNI seq info [547048:547048:8:4294967295] 00000400:00000200:0.0:1788574459.010935:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574459.010940:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574459.010943:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574459.010945:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574459.010948:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574459.010950:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574459.010957:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574459.010958:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574459.010960:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574459.010962:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574459.010964:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574459.010965:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574459.010967:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574459.010970:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516700544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010975:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574459.010977:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516700544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574484 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.010981:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.010982:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574459.010984:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574459.010986:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574459.010988:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574459.010990:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309819049 00000100:00000001:0.0:1788574459.010992:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309822701 00000100:00000001:0.0:1788574459.010993:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309819049 00000100:00000001:0.0:1788574459.010995:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309822701 00000100:00000001:0.0:1788574459.010996:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574459.011000:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.011002:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574459.011004:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574459/real 1788574459] req@ffff8fc4901a5880 x1875440516700544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574459.011010:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574459.011011:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574459.011012:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574459.011014:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574459.011015:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574459.011016:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574459.011018:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516700544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574459.011023:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516700544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.011027:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574459.011028:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.011030:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574459.011030:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574459.011032:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516700544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.011036:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574459.011039:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516700544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.011043:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574459.011044:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.011045:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574459.011046:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.011047:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574459.011048:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574459.011050:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574459.011055:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574459.011056:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574459.011059:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516700544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574459.011064:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516700544:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574459.011066:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.011067:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574459.011069:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516700544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574484 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574459.011073:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574459.011074:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574459.011076:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574459.011077:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574459.011078:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574459.011079:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574459.011080:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574459.011082:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788574459.011083:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574459.011084:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574459.011085:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788574459.011087:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574459.011087:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.011089:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.011091:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574459.011092:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.011093:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574459.011095:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574459.520912:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574459.520920:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574459.520924:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574459.520932:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574459.520937:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574459.520940:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.520955:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574459.520960:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574459.520967:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574460.224988:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574460.224992:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574460.224994:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574460.544969:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574460.544975:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574460.544979:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574460.544984:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574460.544989:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574460.544993:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574460.544996:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574460.545001:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574460.545003:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574461.568897:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574461.568913:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574461.568919:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574461.568928:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574461.568934:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574461.568939:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574461.568949:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574461.568953:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574461.568956:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574462.273004:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574462.273007:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574462.273010:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574462.848921:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309822701, at 4309822720 00000100:00000010:0.0:1788574462.848938:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574463.297001:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574463.297004:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574463.297008:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574463.616928:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574463.616934:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574463.616938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574463.616944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574463.616961:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574463.616973:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574463.616977:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574463.616981:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574463.616986:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574464.128945:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574464.128953:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574464.128956:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574464.128960:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574464.128964:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574464.128971:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15151 00000400:00000200:0.0:1788574464.128978:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574464.128983:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574464.128986:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574464.128988:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.128990:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.128993:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.128995:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574464.128997:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574464.128999:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574464.129001:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574464.129003:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574464.129005:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574464.129007:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574464.129009:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129010:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.129012:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129013:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574464.129015:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574464.129017:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129018:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574464.129020:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574464.129038:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574464.129040:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.129042:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574464.129043:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574464.129048:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574464.129051:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574464.129053:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574464.129055:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.129057:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574464.129059:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574464.129063:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574464.129068:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574464.129074:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574464.129081:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516700672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574464.129089:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574464.129092:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574464.129096:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516700672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574464.129113:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574464.129116:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574464.129118:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574464.129119:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574464.129120:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574464.129122:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574464.129123:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15151 00000400:00000200:0.0:1788574464.129126:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574464.129128:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574464.129129:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574464.129131:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129133:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.129139:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129141:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574464.129143:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574464.129144:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574464.129145:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574464.129147:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574464.129148:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574464.129150:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574464.129153:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129155:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.129158:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129160:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574464.129162:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574464.129167:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129170:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574464.129174:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574464.129191:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574464.129198:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.129204:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574464.129211:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574464.129219:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.129224:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574464.129231:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574464.129236:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574464.129240:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574464.129244:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.129248:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574464.129252:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574464.129257:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574464.129265:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574464.129272:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574464.129281:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516700800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574464.129312:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574464.129315:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574464.129323:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516700800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574464.129340:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574464.129344:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574464.129348:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574464.129352:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574464.129355:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574464.129359:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574464.129365:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15151 00000400:00000200:0.0:1788574464.129369:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574464.129376:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574464.129379:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574464.129385:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129388:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.129393:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129397:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574464.129401:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574464.129404:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574464.129408:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574464.129411:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574464.129416:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574464.129420:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574464.129428:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129430:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.129434:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129438:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574464.129449:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574464.129455:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129459:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574464.129463:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574464.129468:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574464.129473:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.129477:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574464.129479:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574464.129483:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.129486:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574464.129491:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574464.129497:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574464.129500:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574464.129504:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.129509:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574464.129513:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574464.129515:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574464.129521:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574464.129526:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574464.129535:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516700928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574464.129564:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574464.129568:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574464.129579:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516700928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574464.129595:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574464.129600:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574464.129609:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574464.129613:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574464.129616:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574464.129622:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574464.129637:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15151 00000400:00000200:0.0:1788574464.129648:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574464.129656:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574464.129660:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574464.129666:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129671:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.129677:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129681:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574464.129687:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574464.129691:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574464.129695:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574464.129700:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574464.129706:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574464.129711:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574464.129722:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129727:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.129731:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129735:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574464.129741:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574464.129749:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574464.129757:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574464.129762:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574464.129769:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574464.129772:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.129774:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574464.129779:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574464.129783:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.129785:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574464.129789:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574464.129791:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574464.129800:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574464.129853:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.129859:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574464.129862:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574464.129864:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574464.129870:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574464.129876:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574464.129887:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516701056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574464.129901:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574464.129906:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574464.129916:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516701056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574464.129935:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574464.129941:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574464.129951:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574464.129956:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574464.129961:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574464.129968:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574464.129973:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15151 00000400:00000200:0.0:1788574464.129985:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574464.129993:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574464.129999:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574464.130005:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.130010:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.130017:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.130023:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574464.130028:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574464.130031:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574464.130034:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574464.130039:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574464.130044:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574464.130056:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574464.130065:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.130072:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574464.130077:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574464.130080:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574464.130082:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574464.130088:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574464.130094:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574464.130098:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574464.130104:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574464.130112:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.130114:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574464.130117:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574464.130123:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.130127:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574464.130134:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574464.130141:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574464.130148:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574464.130151:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574464.130155:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574464.130159:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574464.130164:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574464.130172:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574464.130193:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574464.130204:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516701184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574464.130220:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574464.130225:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574464.130233:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516701184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574464.130257:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574464.130272:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15161) 00000100:00000001:0.0:1788574464.640925:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574464.640933:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574464.640938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574464.640944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574464.640952:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574464.640956:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574464.640963:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574464.640967:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574464.640972:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574465.344957:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574465.344961:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574465.344963:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574465.664886:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574465.664892:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574465.664895:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574465.664900:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574465.664904:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574465.664909:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574465.664915:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574465.664920:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574465.664924:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574465.664950:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574465.664954:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574465.664958:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574465.664961:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574465.664967:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574465.664976:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574465.664980:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574465.664982:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574465.664985:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574466.688939:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574466.688947:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574466.688952:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574466.688959:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574466.688965:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574466.688970:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574466.688979:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574466.688997:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574466.689004:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574467.393003:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574467.393006:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574467.393008:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574467.712918:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574467.712924:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574467.712928:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574467.712937:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574467.712944:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574467.712952:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574467.712970:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574467.712974:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574467.712978:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574467.968899:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309827821, at 4309827840 00000100:00000010:0.0:1788574467.968914:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab180. 00000020:00000001:0.0:1788574468.416968:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574468.416972:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574468.416976:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574468.736929:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574468.736939:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574468.736944:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574468.736952:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574468.736960:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574468.736966:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574468.736976:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574468.736979:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574468.736983:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574468.737007:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574468.737011:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574468.737014:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574468.737016:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574468.737020:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574468.737023:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574468.737025:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574468.737027:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574468.737030:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574469.760925:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574469.760935:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574469.760939:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574469.760944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574469.760950:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574469.760958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574469.760972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574469.760979:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574469.760991:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574470.465105:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574470.465111:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574470.465115:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574470.785014:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574470.785021:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574470.785027:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574470.785042:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574470.785048:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574470.785052:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574470.785056:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574470.785059:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574470.785062:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574471.808972:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574471.808978:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574471.808981:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574471.808987:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574471.808992:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574471.808997:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574471.809034:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574471.809040:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574471.809045:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574471.809067:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574471.809071:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574471.809074:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574471.809078:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574471.809083:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574471.809088:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574471.809094:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574471.809096:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574471.809122:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574472.513039:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574472.513044:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574472.513047:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574472.832916:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574472.832924:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574472.832929:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574472.832936:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574472.832944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574472.832950:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574472.832956:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574472.832959:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574472.832961:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574472.832975:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574472.832980:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574472.832982:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574472.832985:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574472.832988:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574472.832993:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574472.832996:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574472.832999:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574472.833001:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574473.088924:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309832942, at 4309832960 00000100:00000010:0.0:1788574473.088939:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab480. 00000020:00000001:0.0:1788574473.536987:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574473.536991:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574473.536995:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574473.856898:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574473.856902:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574473.856906:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574473.856910:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574473.856917:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574473.856922:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574473.856944:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574473.856949:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574473.856952:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574473.856973:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574473.856986:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574473.856988:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574473.856991:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574473.856995:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574473.856997:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574473.856999:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574473.857000:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574473.857002:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574474.368948:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574474.368968:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574474.368971:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574474.368974:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574474.368979:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574474.368988:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15161 00000400:00000200:0.0:1788574474.368995:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574474.369001:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574474.369003:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574474.369006:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369008:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574474.369010:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369012:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574474.369016:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574474.369018:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574474.369020:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574474.369023:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574474.369028:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574474.369030:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574474.369034:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369035:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574474.369037:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369038:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574474.369040:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574474.369058:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369060:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574474.369065:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574474.369068:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574474.369071:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369073:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574474.369075:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574474.369080:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574474.369083:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574474.369087:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574474.369088:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369091:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574474.369094:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574474.369101:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574474.369108:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574474.369114:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574474.369121:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516701952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574474.369129:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574474.369137:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574474.369141:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516701952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574474.369158:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574474.369161:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574474.369165:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574474.369172:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574474.369174:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574474.369177:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574474.369180:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15161 00000400:00000200:0.0:1788574474.369185:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574474.369188:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574474.369190:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574474.369199:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369201:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574474.369210:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369213:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574474.369215:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574474.369217:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574474.369218:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574474.369221:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574474.369223:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574474.369226:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574474.369229:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369231:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574474.369233:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369240:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574474.369242:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574474.369245:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369248:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574474.369250:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574474.369255:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574474.369257:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369259:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574474.369264:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574474.369267:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369268:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574474.369272:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574474.369274:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574474.369277:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574474.369278:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369279:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574474.369281:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574474.369283:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574474.369287:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574474.369313:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574474.369319:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516702080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574474.369327:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574474.369329:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574474.369333:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516702080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574474.369342:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574474.369345:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574474.369367:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574474.369369:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574474.369371:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574474.369374:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574474.369376:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15161 00000400:00000200:0.0:1788574474.369380:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574474.369383:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574474.369386:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574474.369388:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369391:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574474.369393:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369395:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574474.369398:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574474.369399:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574474.369401:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574474.369403:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574474.369405:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574474.369407:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574474.369410:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369412:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574474.369419:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369421:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574474.369423:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574474.369426:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369429:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574474.369431:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574474.369434:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574474.369436:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369438:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574474.369441:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574474.369443:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369445:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574474.369449:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574474.369451:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574474.369453:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574474.369455:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369457:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574474.369459:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574474.369460:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574474.369464:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574474.369468:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574474.369473:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516702208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574474.369482:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574474.369484:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574474.369488:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516702208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574474.369505:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574474.369508:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574474.369511:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574474.369513:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574474.369521:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574474.369530:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574474.369533:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15161 00000400:00000200:0.0:1788574474.369537:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574474.369540:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574474.369542:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574474.369545:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369552:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574474.369554:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369557:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574474.369559:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574474.369561:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574474.369562:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574474.369565:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574474.369567:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574474.369570:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574474.369578:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369580:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574474.369582:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369584:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574474.369586:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574474.369589:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369591:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574474.369593:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574474.369596:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574474.369602:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369604:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574474.369607:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574474.369609:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369611:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574474.369618:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574474.369620:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574474.369629:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574474.369630:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369632:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574474.369634:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574474.369636:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574474.369640:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574474.369643:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574474.369653:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516702336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574474.369661:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574474.369663:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574474.369667:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516702336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574474.369680:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574474.369682:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574474.369686:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574474.369687:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574474.369689:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574474.369692:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574474.369698:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15161 00000400:00000200:0.0:1788574474.369702:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574474.369705:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574474.369708:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574474.369711:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369713:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574474.369715:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369717:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574474.369724:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574474.369730:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574474.369731:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574474.369733:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574474.369736:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574474.369738:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574474.369742:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369749:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574474.369751:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369753:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574474.369756:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574474.369759:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574474.369762:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574474.369763:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574474.369766:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574474.369775:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369777:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574474.369779:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574474.369782:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369784:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574474.369787:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574474.369789:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574474.369797:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574474.369799:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574474.369801:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574474.369802:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574474.369804:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574474.369846:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574474.369851:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574474.369857:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516702464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574474.369865:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574474.369867:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574474.369876:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516702464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574474.369886:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574474.369890:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15171) 00000100:00000001:0.0:1788574474.880934:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574474.880942:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574474.880946:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574474.880953:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574474.880958:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574474.880963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574474.880978:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574474.880981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574474.880985:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574475.585033:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574475.585036:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574475.585039:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574475.904895:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574475.904901:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574475.904905:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574475.904911:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574475.904918:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574475.904922:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574475.904926:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574475.904928:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574475.904935:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574476.928950:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574476.928955:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574476.928958:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574476.928964:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574476.928971:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574476.928975:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574476.928977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574476.928979:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574476.928981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574477.632990:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574477.632993:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574477.632996:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574477.952924:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574477.952931:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574477.952936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574477.952945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574477.952953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574477.952960:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574477.952981:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574477.952986:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574477.952992:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574478.208906:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309838061, at 4309838080 00000100:00000010:0.0:1788574478.208916:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abe40. 00000020:00000001:0.0:1788574478.656943:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574478.656946:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574478.656949:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574478.976900:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574478.976908:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574478.976912:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574478.976918:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574478.976924:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574478.976930:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574478.976956:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574478.976959:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574478.976965:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574479.488892:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574479.488904:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574479.488907:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574479.488911:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574479.488916:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574479.488923:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15166 00000400:00000200:0.0:1788574479.488931:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574479.488936:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574479.488938:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574479.488941:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.488944:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574479.488962:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.488970:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574479.488973:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574479.488975:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574479.488977:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574479.488980:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574479.488982:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574479.488984:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574479.488986:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.488987:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574479.488989:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.488990:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574479.488992:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574479.488994:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574479.488996:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574479.488998:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574479.489001:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574479.489003:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489005:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574479.489006:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574479.489012:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574479.489015:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574479.489018:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574479.489020:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489022:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574479.489025:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574479.489028:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574479.489034:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574479.489039:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574479.489046:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516702592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574479.489054:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574479.489073:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574479.489077:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516702592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574479.489094:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574479.489097:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574479.489099:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574479.489100:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574479.489102:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574479.489103:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574479.489105:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15166 00000400:00000200:0.0:1788574479.489108:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574479.489111:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574479.489113:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574479.489115:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489117:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574479.489119:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489120:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574479.489122:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574479.489123:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574479.489125:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574479.489127:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574479.489128:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574479.489129:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574479.489133:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489135:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574479.489139:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489142:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574479.489145:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574479.489149:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489161:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574479.489163:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574479.489170:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574479.489176:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489180:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574479.489185:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574479.489190:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489194:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574479.489197:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574479.489200:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574479.489203:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574479.489205:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489207:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574479.489208:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574479.489213:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574479.489217:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574479.489223:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574479.489230:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516702720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574479.489239:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574479.489241:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574479.489246:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516702720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574479.489256:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574479.489260:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574479.489265:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574479.489267:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574479.489268:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574479.489272:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574479.489275:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15166 00000400:00000200:0.0:1788574479.489282:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574479.489300:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574479.489303:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574479.489309:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489314:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574479.489318:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489323:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574479.489325:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574479.489326:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574479.489329:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574479.489332:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574479.489334:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574479.489336:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574479.489340:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489344:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574479.489346:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489348:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574479.489365:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574479.489370:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489375:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574479.489379:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574479.489384:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574479.489388:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489392:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574479.489395:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574479.489400:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489404:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574479.489410:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574479.489415:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574479.489422:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574479.489425:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489429:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574479.489438:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574479.489443:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574479.489450:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574479.489455:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574479.489463:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516702848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574479.489473:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574479.489477:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574479.489485:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516702848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574479.489496:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574479.489500:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574479.489505:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574479.489509:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574479.489510:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574479.489513:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574479.489516:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15166 00000400:00000200:0.0:1788574479.489522:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574479.489526:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574479.489529:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574479.489535:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489540:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574479.489543:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489547:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574479.489551:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574479.489552:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574479.489554:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574479.489557:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574479.489560:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574479.489565:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574479.489577:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489580:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574479.489584:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489589:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574479.489592:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574479.489598:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489600:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574479.489603:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574479.489605:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574479.489607:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489610:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574479.489613:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574479.489618:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489620:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574479.489625:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574479.489629:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574479.489633:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574479.489636:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489638:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574479.489641:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574479.489645:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574479.489649:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574479.489654:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574479.489661:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516702976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574479.489674:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574479.489678:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574479.489686:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516702976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574479.489697:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574479.489698:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574479.489709:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574479.489711:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574479.489714:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574479.489719:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574479.489724:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15166 00000400:00000200:0.0:1788574479.489732:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574479.489737:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574479.489742:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574479.489747:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489751:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574479.489759:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489765:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574479.489772:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574479.489776:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574479.489782:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574479.489788:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574479.489794:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574479.489799:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574479.489831:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489835:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574479.489837:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489840:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574479.489843:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574479.489848:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574479.489854:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574479.489859:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574479.489866:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574479.489871:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489881:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574479.489886:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574479.489896:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489897:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574479.489900:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574479.489904:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574479.489908:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574479.489911:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574479.489913:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574479.489917:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574479.489922:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574479.489929:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574479.489935:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574479.489942:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516703104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574479.489952:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574479.489954:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574479.489957:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516703104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574479.489964:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574479.489971:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15177) 00000100:00000001:0.0:1788574480.000888:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574480.000896:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574480.000900:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574480.000908:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574480.000916:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574480.000923:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574480.000931:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574480.000935:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574480.000940:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574480.000971:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574480.000976:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574480.000980:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574480.000983:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574480.000989:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574480.000992:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574480.001006:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574480.001009:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574480.001012:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574480.704935:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574480.704938:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574480.704940:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574482.049040:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574482.049057:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574482.049058:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574482.049061:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574482.049063:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574482.049065:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574482.049094:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574482.049096:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574482.049097:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574482.753004:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574482.753007:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574482.753010:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574483.328895:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309843181, at 4309843200 00000100:00000010:0.0:1788574483.328904:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab1e0. 00000020:00000001:0.0:1788574483.776993:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574483.776996:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574483.776998:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574484.096946:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574484.096951:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574484.096955:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574484.096960:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574484.096964:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574484.096968:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574484.096993:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574484.096998:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574484.097007:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574485.120989:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574485.121001:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574485.121006:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574485.121015:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574485.121020:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574485.121046:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574485.121050:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574485.121053:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574485.121057:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574485.824982:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574485.824987:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574485.824991:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574486.144949:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574486.144955:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574486.144966:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574486.144972:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574486.144978:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574486.144983:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574486.145000:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574486.145004:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574486.145007:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574487.168959:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574487.168965:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574487.168968:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574487.168973:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574487.168977:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574487.168982:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574487.168991:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574487.168994:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574487.168997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574487.872952:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574487.872955:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574487.872958:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574488.192921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574488.192929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574488.192932:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574488.192937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574488.192942:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574488.192949:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574488.192956:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574488.192961:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574488.192965:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574488.448893:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309848301, at 4309848320 00000100:00000010:0.0:1788574488.448916:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9aba20. 00000020:00000001:0.0:1788574488.897097:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574488.897101:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574488.897105:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574489.216975:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574489.216986:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574489.216996:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574489.217014:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574489.217024:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574489.217038:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574489.217070:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574489.217075:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574489.217079:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574489.728972:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574489.728985:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574489.728989:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574489.728993:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574489.728999:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574489.729008:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15177 00000400:00000200:0.0:1788574489.729019:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574489.729027:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574489.729031:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574489.729035:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729039:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574489.729042:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729045:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574489.729050:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574489.729052:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574489.729055:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574489.729059:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574489.729062:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574489.729064:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574489.729067:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729093:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574489.729095:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729097:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574489.729101:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574489.729105:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729107:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574489.729111:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574489.729115:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574489.729119:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.729121:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574489.729124:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574489.729131:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574489.729135:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574489.729140:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574489.729142:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.729146:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574489.729150:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574489.729155:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574489.729164:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574489.729172:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574489.729183:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516703872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574489.729196:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574489.729201:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574489.729206:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516703872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574489.729238:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574489.729241:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574489.729246:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574489.729247:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574489.729249:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574489.729252:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574489.729259:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15177 00000400:00000200:0.0:1788574489.729264:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574489.729267:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574489.729270:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574489.729273:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729275:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574489.729278:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729280:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574489.729282:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574489.729284:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574489.729285:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574489.729288:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574489.729290:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574489.729292:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574489.729295:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729298:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574489.729300:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729303:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574489.729306:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574489.729310:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729313:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574489.729315:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574489.729322:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574489.729326:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.729332:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574489.729339:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574489.729344:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.729349:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574489.729355:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574489.729361:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574489.729365:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574489.729374:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.729380:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574489.729383:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574489.729388:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574489.729394:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574489.729401:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574489.729411:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516704000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574489.729426:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574489.729431:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574489.729440:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516704000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574489.729454:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574489.729457:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574489.729492:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574489.729496:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574489.729501:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574489.729506:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574489.729512:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15177 00000400:00000200:0.0:1788574489.729520:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574489.729525:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574489.729529:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574489.729537:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729540:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574489.729546:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729550:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574489.729554:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574489.729558:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574489.729561:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574489.729563:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574489.729567:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574489.729577:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574489.729585:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729591:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574489.729596:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729601:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574489.729607:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574489.729613:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729615:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574489.729617:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574489.729621:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574489.729624:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.729626:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574489.729629:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574489.729631:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.729635:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574489.729640:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574489.729644:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574489.729646:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574489.729647:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.729652:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574489.729654:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574489.729657:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574489.729662:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574489.729668:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574489.729677:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516704128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574489.729692:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574489.729696:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574489.729703:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516704128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574489.729721:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574489.729735:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574489.729740:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574489.729743:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574489.729746:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574489.729751:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574489.729757:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15177 00000400:00000200:0.0:1788574489.729763:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574489.729769:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574489.729774:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574489.729777:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729781:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574489.729785:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729790:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574489.729794:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574489.729797:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574489.729800:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574489.729804:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574489.729868:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574489.729871:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574489.729875:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729877:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574489.729880:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729882:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574489.729885:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574489.729887:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574489.729890:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574489.729892:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574489.729895:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574489.729899:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.729904:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574489.729916:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574489.729923:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.729927:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574489.729933:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574489.729939:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574489.729944:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574489.729947:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.729951:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574489.729954:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574489.729959:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574489.729967:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574489.729972:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574489.729980:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516704256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574489.729991:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574489.729995:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574489.730003:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516704256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574489.730020:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574489.730025:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574489.730030:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574489.730034:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574489.730037:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574489.730044:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574489.730048:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15177 00000400:00000200:0.0:1788574489.730058:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574489.730065:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574489.730069:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574489.730073:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.730079:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574489.730085:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.730097:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574489.730102:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574489.730105:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574489.730109:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574489.730113:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574489.730118:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574489.730122:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574489.730131:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.730136:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574489.730141:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574489.730147:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574489.730153:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574489.730159:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574489.730164:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574489.730167:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574489.730173:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574489.730178:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.730182:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574489.730186:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574489.730191:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.730194:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574489.730202:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574489.730207:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574489.730213:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574489.730216:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574489.730221:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574489.730225:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574489.730229:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574489.730235:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574489.730242:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574489.730253:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516704384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574489.730299:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574489.730304:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574489.730314:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516704384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574489.730333:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574489.730341:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15187) 00000100:00000001:0.0:1788574490.240940:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574490.240946:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574490.240948:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574490.240952:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574490.240965:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574490.240970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574490.240975:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574490.240978:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574490.240984:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574490.241033:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574490.241040:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574490.241045:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574490.241060:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574490.241066:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574490.241070:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574490.241077:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574490.241078:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574490.241080:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574490.944981:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574490.944985:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574490.944987:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574491.264925:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574491.264931:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574491.264934:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574491.264939:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574491.264942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574491.264945:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574491.264961:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574491.264983:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574491.264993:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574492.288963:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574492.288968:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574492.288972:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574492.288977:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574492.288983:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574492.288988:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574492.289005:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574492.289010:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574492.289014:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574492.993092:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574492.993096:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574492.993099:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574493.312950:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574493.312958:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574493.312962:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574493.312970:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574493.312977:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574493.312985:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574493.312994:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574493.312997:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574493.313000:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574493.568933:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309853422, at 4309853440 00000100:00000010:0.0:1788574493.568957:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab2a0. 00000020:00000001:0.0:1788574494.016952:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574494.016955:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574494.016958:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574494.336935:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574494.336945:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574494.336950:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574494.336963:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574494.336973:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574494.336983:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574494.336996:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574494.337007:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574494.337015:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574494.848919:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574494.848928:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574494.848931:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574494.848935:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574494.848939:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574494.848946:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15182 00000400:00000200:0.0:1788574494.848957:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574494.848966:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574494.848970:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574494.848973:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.848976:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574494.848979:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.848982:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574494.848985:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574494.848987:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574494.848989:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574494.848991:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574494.848993:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574494.848995:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574494.848998:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.848999:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574494.849001:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849002:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574494.849005:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574494.849007:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849009:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574494.849011:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574494.849013:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574494.849015:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849016:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574494.849017:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574494.849037:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574494.849041:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574494.849044:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574494.849045:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849048:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574494.849050:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574494.849062:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574494.849068:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574494.849075:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574494.849083:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516704512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574494.849091:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574494.849094:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574494.849098:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516704512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574494.849127:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574494.849133:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574494.849139:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574494.849143:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574494.849145:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574494.849149:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574494.849153:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15182 00000400:00000200:0.0:1788574494.849157:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574494.849161:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574494.849163:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574494.849166:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849171:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574494.849173:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849178:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574494.849180:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574494.849182:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574494.849192:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574494.849195:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574494.849199:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574494.849203:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574494.849210:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849215:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574494.849219:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849221:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574494.849227:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574494.849229:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849234:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574494.849235:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574494.849243:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574494.849246:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849249:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574494.849252:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574494.849257:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849259:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574494.849264:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574494.849266:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574494.849271:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574494.849272:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849273:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574494.849274:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574494.849276:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574494.849279:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574494.849284:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574494.849291:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516704640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574494.849303:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574494.849307:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574494.849312:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516704640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574494.849328:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574494.849331:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574494.849334:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574494.849337:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574494.849341:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574494.849346:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574494.849351:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15182 00000400:00000200:0.0:1788574494.849360:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574494.849367:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574494.849372:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574494.849380:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849386:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574494.849390:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849394:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574494.849396:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574494.849398:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574494.849401:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574494.849403:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574494.849408:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574494.849412:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574494.849415:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849417:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574494.849426:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849430:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574494.849432:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574494.849437:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849441:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574494.849449:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574494.849454:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574494.849458:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849460:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574494.849463:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574494.849468:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849472:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574494.849475:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574494.849479:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574494.849482:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574494.849484:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849488:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574494.849490:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574494.849494:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574494.849515:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574494.849518:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574494.849524:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516704768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574494.849530:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574494.849534:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574494.849541:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516704768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574494.849556:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574494.849559:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574494.849564:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574494.849566:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574494.849568:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574494.849573:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574494.849576:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15182 00000400:00000200:0.0:1788574494.849582:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574494.849587:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574494.849592:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574494.849605:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849610:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574494.849615:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849617:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574494.849621:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574494.849624:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574494.849627:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574494.849630:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574494.849633:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574494.849637:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574494.849642:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849645:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574494.849647:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849649:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574494.849651:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574494.849654:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849656:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574494.849658:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574494.849661:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574494.849666:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849669:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574494.849673:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574494.849675:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849676:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574494.849679:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574494.849683:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574494.849688:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574494.849690:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849693:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574494.849694:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574494.849696:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574494.849701:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574494.849710:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574494.849718:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516704896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574494.849725:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574494.849727:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574494.849732:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516704896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574494.849744:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574494.849747:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574494.849751:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574494.849753:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574494.849754:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574494.849756:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574494.849761:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15182 00000400:00000200:0.0:1788574494.849765:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574494.849770:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574494.849775:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574494.849778:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849781:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574494.849786:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849791:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574494.849794:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574494.849796:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574494.849799:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574494.849802:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574494.849806:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574494.849826:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574494.849829:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849830:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574494.849836:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849838:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574494.849840:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574494.849844:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574494.849849:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574494.849854:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574494.849857:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574494.849859:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849861:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574494.849866:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574494.849870:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849874:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574494.849877:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574494.849880:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574494.849884:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574494.849886:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574494.849889:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574494.849891:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574494.849894:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574494.849900:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574494.849905:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574494.849913:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516705024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574494.849921:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574494.849922:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574494.849928:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516705024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574494.849940:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574494.849944:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15192) 00000100:00000001:0.0:1788574495.360965:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574495.360970:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574495.360971:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574495.360984:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574495.360989:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574495.360992:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574495.360996:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574495.360998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574495.361004:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574496.064975:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574496.064979:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574496.064982:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574496.384931:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574496.384936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574496.384941:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574496.384948:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574496.384954:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574496.384957:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574496.384965:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574496.384968:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574496.384972:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574496.385017:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574496.385020:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574496.385021:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574496.385023:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574496.385026:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574496.385029:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574496.385031:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574496.385032:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574496.385034:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574497.408904:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574497.408911:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574497.408915:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574497.408921:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574497.408925:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574497.408930:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574497.408932:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574497.408934:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574497.408936:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574498.112954:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574498.112957:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574498.112978:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574498.432966:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574498.432973:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574498.432979:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574498.432988:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574498.432995:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574498.432999:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574498.433002:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574498.433005:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574498.433010:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574498.688905:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309858541, at 4309858560 00000100:00000010:0.0:1788574498.688920:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab540. 00000020:00000001:0.0:1788574499.136986:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574499.136990:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574499.136992:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574499.456926:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574499.456934:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574499.456940:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574499.456949:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574499.456956:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574499.456963:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574499.456979:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574499.456985:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574499.456992:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574499.968903:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574499.968912:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574499.968914:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574499.968917:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574499.968922:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574499.968929:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15187 00000400:00000200:0.0:1788574499.968937:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574499.968942:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574499.968944:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574499.968946:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.968965:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574499.968967:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.968969:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574499.968972:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574499.968974:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574499.968976:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574499.968978:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574499.968980:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574499.968982:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574499.968983:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.968985:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574499.968986:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.968987:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574499.968989:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574499.968992:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574499.968993:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574499.968996:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574499.968998:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574499.969000:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.969001:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574499.969003:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574499.969009:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574499.969012:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574499.969015:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574499.969016:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.969019:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574499.969021:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574499.969024:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574499.969030:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574499.969036:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574499.969043:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516705152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574499.969057:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574499.969061:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574499.969064:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516705152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574499.969083:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574499.969087:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574499.969090:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574499.969092:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574499.969094:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574499.969097:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574499.969099:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15187 00000400:00000200:0.0:1788574499.969104:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574499.969108:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574499.969111:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574499.969115:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969117:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574499.969120:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969123:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574499.969126:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574499.969130:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574499.969135:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574499.969139:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574499.969143:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574499.969147:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574499.969154:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969160:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574499.969165:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969170:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574499.969181:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574499.969190:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969196:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574499.969201:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574499.969209:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574499.969215:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.969219:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574499.969224:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574499.969230:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.969235:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574499.969243:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574499.969247:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574499.969253:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574499.969258:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.969262:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574499.969264:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574499.969268:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574499.969275:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574499.969281:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574499.969293:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516705280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574499.969303:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574499.969306:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574499.969311:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516705280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574499.969320:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574499.969325:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574499.969328:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574499.969329:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574499.969331:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574499.969335:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574499.969346:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15187 00000400:00000200:0.0:1788574499.969353:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574499.969360:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574499.969364:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574499.969370:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969374:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574499.969378:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969382:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574499.969385:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574499.969387:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574499.969389:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574499.969391:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574499.969396:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574499.969401:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574499.969408:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969410:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574499.969414:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969417:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574499.969420:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574499.969425:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969427:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574499.969430:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574499.969433:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574499.969438:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.969440:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574499.969443:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574499.969445:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.969447:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574499.969450:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574499.969451:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574499.969453:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574499.969458:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.969462:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574499.969464:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574499.969466:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574499.969471:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574499.969475:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574499.969482:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516705408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574499.969490:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574499.969494:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574499.969501:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516705408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574499.969515:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574499.969519:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574499.969524:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574499.969539:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574499.969543:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574499.969547:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574499.969552:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15187 00000400:00000200:0.0:1788574499.969562:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574499.969570:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574499.969576:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574499.969583:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969588:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574499.969593:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969598:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574499.969603:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574499.969605:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574499.969607:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574499.969611:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574499.969615:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574499.969626:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574499.969635:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969641:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574499.969647:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969652:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574499.969657:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574499.969665:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969672:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574499.969677:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574499.969682:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574499.969688:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.969694:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574499.969700:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574499.969705:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.969710:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574499.969716:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574499.969723:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574499.969730:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574499.969734:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.969745:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574499.969749:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574499.969755:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574499.969762:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574499.969770:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574499.969782:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516705536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574499.969798:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574499.969803:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574499.969833:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516705536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574499.969843:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574499.969853:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574499.969859:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574499.969863:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574499.969866:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574499.969872:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574499.969878:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15187 00000400:00000200:0.0:1788574499.969887:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574499.969893:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574499.969899:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574499.969906:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969912:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574499.969917:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969923:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574499.969928:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574499.969931:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574499.969935:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574499.969939:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574499.969944:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574499.969948:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574499.969954:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969957:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574499.969962:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969966:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574499.969971:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574499.969977:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574499.969982:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574499.969985:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574499.969990:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574499.969997:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.970003:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574499.970013:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574499.970018:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.970022:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574499.970026:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574499.970031:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574499.970037:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574499.970038:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574499.970042:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574499.970045:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574499.970049:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574499.970055:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574499.970062:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574499.970073:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516705664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574499.970092:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574499.970098:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574499.970107:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516705664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574499.970125:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574499.970133:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15197) 00000100:00000001:0.0:1788574500.480949:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574500.480959:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574500.480966:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574500.480980:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574500.480991:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574500.481003:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574500.481013:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574500.481019:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574500.481026:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574500.481065:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574500.481069:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574500.481071:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574500.481075:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574500.481078:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574500.481087:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574500.481090:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574500.481092:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574500.481094:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574501.185003:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574501.185006:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574501.185009:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574501.504907:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574501.504914:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574501.504917:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574501.504923:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574501.504931:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574501.504937:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574501.504943:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574501.504947:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574501.504950:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574501.504971:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574501.504976:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574501.504978:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574501.504982:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574501.504985:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574501.504989:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574501.504992:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574501.504994:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574501.504996:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574502.528931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574502.528936:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574502.528947:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574502.528952:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574502.528958:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574502.528962:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574502.528965:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574502.528967:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574502.528969:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574502.529000:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574502.529002:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574502.529003:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574502.529005:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574502.529022:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574502.529025:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574502.529028:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574502.529030:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574502.529031:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574503.041316:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574503.041323:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574503.041329:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574503.041341:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574499/real 1788574503] req@ffff8fc4b80c1c00 x1875440516705152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574503.041360:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574503.041363:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574503.041369:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a0c08. 00000400:00000200:0.0:1788574503.041377:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed2a8 00000400:00000010:0.0:1788574503.041380:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed2a8. 00000100:00000001:0.0:1788574503.041386:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574503.041393:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c1c00 x1875440516705152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574503.041414:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c1c00 x1875440516705152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.041425:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574503.041429:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.041436:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574503.041439:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574503.041452:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c1c00 x1875440516705152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.041466:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574503.041477:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c1c00 x1875440516705152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.041488:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574503.041496:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.041510:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574503.041513:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.041519:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574503.041524:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574503.041532:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574503.041541:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574503.041561:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574503.041567:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c1c00 x1875440516705152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574503.041576:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516705152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574503.041583:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574503.041588:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574503.041596:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c1c00 x1875440516705152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.041611:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574503.041616:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574503.041622:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574503.041630:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574503.041635:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574503.041639:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574503.041643:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574503.041648:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574503.041655:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574503.041658:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574503.041661:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c1c00. 00000100:00000001:0.0:1788574503.041666:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574503.041669:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.041673:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574503.041678:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574503.041680:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574503.041684:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574503.041692:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574499/real 1788574503] req@ffff8fc4b80c2680 x1875440516705536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574503.041727:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574503.041729:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574503.041732:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a0cb8. 00000400:00000200:0.0:1788574503.041736:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed330 00000400:00000010:0.0:1788574503.041738:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed330. 00000100:00000001:0.0:1788574503.041741:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574503.041745:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c2680 x1875440516705536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574503.041754:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c2680 x1875440516705536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.041760:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574503.041762:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.041764:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574503.041765:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574503.041769:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c2680 x1875440516705536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.041788:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574503.041793:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c2680 x1875440516705536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.041800:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574503.041803:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.041804:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574503.041805:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.041843:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574503.041846:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574503.041849:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574503.041854:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574503.041857:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574503.041868:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c2680 x1875440516705536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574503.041878:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516705536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574503.041882:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574503.041886:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574499/real 1788574503] req@ffff8fc4b80c1f80 x1875440516705664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574503.041895:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574503.041897:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574503.041899:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a09a0. 00000400:00000200:0.0:1788574503.041903:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed3b8 00000400:00000010:0.0:1788574503.041905:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed3b8. 00000100:00000001:0.0:1788574503.041909:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574503.041912:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c1f80 x1875440516705664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574503.041919:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c1f80 x1875440516705664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.041941:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574503.041943:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.041946:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574503.041947:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574503.041951:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c1f80 x1875440516705664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.041958:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574503.041962:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c1f80 x1875440516705664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.041969:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574503.041971:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.041981:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574503.041983:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.041985:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574503.041988:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574503.041990:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574503.041995:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574503.041997:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574503.042002:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c1f80 x1875440516705664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574503.042011:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516705664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574503.042015:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574503.042019:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574499/real 1788574503] req@ffff8fc4b80c0700 x1875440516705280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574503.042026:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574503.042028:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574503.042030:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a0840. 00000400:00000200:0.0:1788574503.042033:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76edee0 00000400:00000010:0.0:1788574503.042034:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76edee0. 00000100:00000001:0.0:1788574503.042037:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574503.042039:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c0700 x1875440516705280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574503.042047:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c0700 x1875440516705280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.042053:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574503.042054:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.042056:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574503.042057:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574503.042061:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c0700 x1875440516705280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.042080:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574503.042085:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c0700 x1875440516705280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.042091:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574503.042093:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.042095:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574503.042097:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.042099:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574503.042101:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574503.042104:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574503.042108:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574503.042111:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574503.042115:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c0700 x1875440516705280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574503.042122:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516705280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574503.042126:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574503.042129:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574499/real 1788574503] req@ffff8fc4b80c2300 x1875440516705408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574503.042139:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574503.042140:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574503.042142:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a0948. 00000400:00000200:0.0:1788574503.042144:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76edaa0 00000400:00000010:0.0:1788574503.042146:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76edaa0. 00000100:00000001:0.0:1788574503.042148:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574503.042152:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c2300 x1875440516705408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574503.042171:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c2300 x1875440516705408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.042178:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574503.042179:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.042181:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574503.042182:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574503.042186:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c2300 x1875440516705408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.042193:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574503.042197:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c2300 x1875440516705408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.042203:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574503.042206:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.042207:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574503.042208:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.042210:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574503.042213:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574503.042216:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574503.042219:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574503.042222:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574503.042226:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c2300 x1875440516705408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574503.042234:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516705408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574503.042238:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.042240:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574503.042244:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c2680 x1875440516705536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.042262:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574503.042264:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574503.042267:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788574503.042270:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574503.042272:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574503.042273:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574503.042276:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574503.042277:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574503.042280:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574503.042281:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574503.042282:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c2680. 00000100:00000001:0.0:1788574503.042284:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574503.042285:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.042286:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574503.042290:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c1f80 x1875440516705664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.042297:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574503.042299:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574503.042300:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788574503.042303:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574503.042304:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574503.042305:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574503.042308:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574503.042309:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574503.042312:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574503.042313:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574503.042314:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c1f80. 00000100:00000001:0.0:1788574503.042317:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574503.042318:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.042320:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574503.042323:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c0700 x1875440516705280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.042330:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574503.042331:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574503.042339:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788574503.042342:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574503.042344:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574503.042345:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574503.042347:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574503.042349:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574503.042351:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574503.042352:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574503.042353:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c0700. 00000100:00000001:0.0:1788574503.042356:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574503.042357:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.042359:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574503.042362:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c2300 x1875440516705408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574524 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574503.042369:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574503.042370:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574503.042371:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788574503.042374:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574503.042375:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574503.042376:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574503.042378:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574503.042380:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574503.042383:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574503.042384:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574503.042386:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c2300. 00000100:00000001:0.0:1788574503.042389:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574503.042391:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.042393:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.042402:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574503.042405:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574503.042408:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574503.042409:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574503.232968:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574503.232972:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574503.232976:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574503.552927:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574503.552935:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574503.552954:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574503.552958:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574503.552964:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574503.552968:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.552982:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574503.552985:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574503.552988:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574503.808908:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309863661, at 4309863680 00000100:00000010:0.0:1788574503.808928:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab720. 00000020:00000001:0.0:1788574504.256981:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574504.256984:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574504.256987:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574504.576895:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574504.576903:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574504.576908:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574504.576914:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574504.576921:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574504.576927:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574504.576934:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574504.576937:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574504.576939:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574504.576951:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574504.576956:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574504.576960:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574504.576966:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574504.576969:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574504.576972:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574504.576974:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574504.576976:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574504.576978:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574505.088901:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574505.088923:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574505.088926:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574505.088930:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574505.088935:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574505.088942:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15192 00000400:00000200:0.0:1788574505.088974:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574505.088980:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574505.088984:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574505.088988:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.088991:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574505.088993:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.088995:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574505.088999:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574505.089000:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574505.089002:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574505.089005:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574505.089007:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574505.089009:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574505.089011:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089012:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574505.089014:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089015:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574505.089017:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574505.089020:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089023:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574505.089026:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574505.089029:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574505.089033:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089035:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574505.089037:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574505.089045:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574505.089049:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574505.089061:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574505.089063:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089066:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574505.089069:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574505.089074:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574505.089091:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574505.089098:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574505.089109:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516705792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574505.089120:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574505.089124:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574505.089128:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516705792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574505.089171:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574505.089174:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574505.089178:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574505.089180:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574505.089181:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574505.089184:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574505.089187:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15192 00000400:00000200:0.0:1788574505.089192:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574505.089195:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574505.089198:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574505.089201:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089203:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574505.089205:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089207:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574505.089209:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574505.089211:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574505.089213:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574505.089215:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574505.089217:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574505.089220:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574505.089223:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089231:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574505.089234:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089237:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574505.089240:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574505.089242:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089245:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574505.089247:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574505.089253:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574505.089256:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089258:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574505.089262:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574505.089265:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089267:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574505.089272:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574505.089275:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574505.089279:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574505.089281:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089282:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574505.089284:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574505.089286:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574505.089290:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574505.089294:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574505.089300:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516705920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574505.089309:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574505.089311:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574505.089315:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516705920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574505.089324:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574505.089326:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574505.089329:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574505.089337:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574505.089338:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574505.089341:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574505.089344:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15192 00000400:00000200:0.0:1788574505.089349:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574505.089353:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574505.089355:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574505.089358:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089360:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574505.089362:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089364:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574505.089366:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574505.089367:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574505.089369:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574505.089372:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574505.089373:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574505.089375:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574505.089379:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089380:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574505.089382:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089385:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574505.089387:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574505.089389:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089391:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574505.089393:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574505.089396:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574505.089398:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089400:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574505.089403:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574505.089405:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089412:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574505.089415:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574505.089417:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574505.089427:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574505.089428:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089430:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574505.089431:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574505.089433:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574505.089437:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574505.089442:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574505.089447:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516706048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574505.089454:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574505.089456:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574505.089459:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516706048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574505.089467:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574505.089469:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574505.089472:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574505.089474:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574505.089476:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574505.089478:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574505.089481:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15192 00000400:00000200:0.0:1788574505.089485:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574505.089488:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574505.089490:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574505.089493:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089496:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574505.089497:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089500:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574505.089502:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574505.089509:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574505.089511:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574505.089513:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574505.089515:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574505.089518:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574505.089521:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089523:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574505.089525:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089527:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574505.089529:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574505.089532:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089535:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574505.089536:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574505.089540:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574505.089542:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089544:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574505.089546:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574505.089548:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089550:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574505.089553:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574505.089574:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574505.089577:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574505.089578:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089580:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574505.089581:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574505.089583:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574505.089588:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574505.089591:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574505.089597:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516706176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574505.089604:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574505.089606:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574505.089617:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516706176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574505.089634:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574505.089640:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574505.089646:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574505.089649:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574505.089653:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574505.089659:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574505.089665:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15192 00000400:00000200:0.0:1788574505.089674:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574505.089679:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574505.089684:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574505.089698:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089709:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574505.089714:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089718:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574505.089721:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574505.089723:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574505.089724:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574505.089728:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574505.089732:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574505.089736:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574505.089743:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089749:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574505.089757:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089762:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574505.089766:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574505.089772:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574505.089775:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574505.089786:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574505.089791:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574505.089796:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089800:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574505.089804:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574505.089851:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089855:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574505.089864:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574505.089869:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574505.089873:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574505.089876:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.089881:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574505.089884:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574505.089886:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574505.089893:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574505.089899:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574505.089908:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516706304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574505.089921:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574505.089925:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574505.089933:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516706304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574505.089948:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574505.089956:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15202) 00000100:00000001:0.0:1788574505.600914:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574505.600922:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574505.600926:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574505.600933:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574505.600937:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574505.600945:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574505.600950:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574505.600954:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574505.600957:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574505.600983:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574505.601005:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574505.601007:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574505.601009:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574505.601011:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574505.601013:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574505.601015:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574505.601017:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574505.601019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574506.304990:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574506.304993:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574506.304995:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574506.624930:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574506.624937:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574506.624942:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574506.624951:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574506.624958:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574506.624963:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574506.624968:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574506.624970:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574506.624974:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574507.648916:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574507.648927:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574507.648932:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574507.648942:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574507.648950:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574507.648959:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574507.648984:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574507.648988:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574507.648991:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574508.352953:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574508.352956:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574508.352959:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574508.672906:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574508.672914:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574508.672919:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574508.672927:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574508.672934:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574508.672940:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574508.672969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574508.672971:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574508.672974:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574508.928903:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309868782, at 4309868800 00000100:00000010:0.0:1788574508.928921:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abb40. 00000020:00000001:0.0:1788574509.377037:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574509.377041:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574509.377044:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574509.696955:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574509.696965:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574509.696974:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574509.696982:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574509.696989:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574509.696996:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574509.697006:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574509.697016:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574509.697023:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574509.697066:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574509.697069:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574509.697070:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574509.697072:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574509.697075:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574509.697088:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574509.697094:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574509.697097:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574509.697101:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574510.720943:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574510.720951:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574510.720953:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574510.720960:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574510.720965:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574510.720971:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574510.720980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574510.720987:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574510.720996:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574511.425006:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574511.425009:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574511.425018:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574511.744981:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574511.744991:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574511.744995:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574511.745010:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574511.745014:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574511.745019:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574511.745035:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574511.745040:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574511.745045:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574512.768941:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574512.768954:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574512.768958:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574512.768966:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574512.768974:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574512.768983:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574512.768997:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574512.769017:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574512.769025:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574513.473023:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574513.473027:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574513.473030:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574513.792991:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574513.792997:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574513.793003:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574513.793011:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574513.793019:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574513.793028:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574513.793038:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574513.793045:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574513.793052:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574514.048919:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309873901, at 4309873920 00000100:00000010:0.0:1788574514.048935:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab600. 00000020:00000001:0.0:1788574514.497090:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574514.497094:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574514.497098:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574514.816901:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574514.816926:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574514.816929:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574514.816937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574514.816944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574514.816949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574514.816954:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574514.816958:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574514.816962:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574514.816995:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574514.816998:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574514.817000:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574514.817002:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574514.817005:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574514.817007:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574514.817009:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574514.817010:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574514.817013:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.329261:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574515.329269:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574515.329274:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574515.329285:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516707072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.329306:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574515.329312:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574515.329320:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574515.329325:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574515.329333:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574515.329342:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516707072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574515.329352:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574515.329359:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574515.329364:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574515.329371:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574515.329380:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574515.329387:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574515.329401:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574515.329410:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574515.329432:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574515.329444:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516707072, portal 25 00000100:00000001:0.0:1788574515.329451:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574515.329457:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788574515.329474:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516707072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.329495:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574515.329505:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574515.329514:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574515.329522:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516707072, offset 0 00000400:00000200:0.0:1788574515.329537:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574515.329553:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574515.329566:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574515.329579:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574515.329588:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574515.329597:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574515.329605:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574515.329631:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547176:547176:256:4294967295] 192.168.202.128@tcp LPNI seq info [547099:547099:8:4294967295] 00000400:00000200:0.0:1788574515.329645:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574515.329672:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574515.329690:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574515.329699:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574515.329716:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574515.329725:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574515.329761:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574515.329766:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574515.329776:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574515.329787:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574515.329798:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574515.329805:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574515.329871:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574515.329890:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516707072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.329929:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574515.329942:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516707072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.329963:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.329968:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574515.329982:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574515.329994:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574515.330001:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309879022 00000100:00000010:0.0:1788574515.330011:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574515.330024:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574515.330035:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309879022 00000100:00000010:0.0:1788574515.330044:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788574515.330054:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574515.330062:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574515.330068:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.330071:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574515.330079:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574515/real 1788574515] req@ffff8fc4ac821880 x1875440516707072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574515.330092:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574515.330095:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574515.330098:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574515.330102:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574515.330104:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574515.330107:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574515.330112:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516707072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574515.330122:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516707072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330131:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574515.330140:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.330142:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574515.330145:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574515.330150:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516707072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330159:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574515.330164:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516707072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330173:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574515.330177:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.330179:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574515.330180:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.330183:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574515.330185:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574515.330189:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574515.330193:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574515.330196:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574515.330202:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516707072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574515.330212:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516707072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574515.330218:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.330221:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574515.330225:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516707072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330233:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574515.330236:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574515.330239:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574515.330244:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574515.330246:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574515.330255:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574515.330258:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574515.330261:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788574515.330266:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574515.330268:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574515.330270:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788574515.330274:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574515.330275:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.330278:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.330287:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574515.330289:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.330292:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574515.330293:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574515.330295:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574515.330300:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516707200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330309:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574515.330311:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574515.330313:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574515.330316:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574515.330318:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574515.330322:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516707200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574515.330326:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574515.330328:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574515.330330:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574515.330332:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574515.330336:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574515.330338:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574515.330343:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574515.330346:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574515.330350:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574515.330354:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516707200, portal 10 00000100:00000001:0.0:1788574515.330356:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574515.330358:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788574515.330366:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516707200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330379:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574515.330382:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574515.330385:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574515.330389:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516707200, offset 0 00000400:00000200:0.0:1788574515.330394:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574515.330399:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574515.330403:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574515.330406:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574515.330409:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574515.330412:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574515.330415:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574515.330418:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547177:547177:256:4294967295] 192.168.202.128@tcp LPNI seq info [547100:547100:8:4294967295] 00000400:00000200:0.0:1788574515.330423:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574515.330431:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574515.330436:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574515.330441:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574515.330446:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574515.330449:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574515.330462:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574515.330464:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574515.330468:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574515.330471:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574515.330475:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574515.330477:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574515.330480:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574515.330484:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516707200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330493:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574515.330496:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516707200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330517:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.330520:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574515.330525:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574515.330529:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574515.330534:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574515.330536:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309875219 00000100:00000001:0.0:1788574515.330539:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309879022 00000100:00000001:0.0:1788574515.330542:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309875219 00000100:00000001:0.0:1788574515.330544:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309879022 00000100:00000001:0.0:1788574515.330547:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574515.330549:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.330551:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574515.330556:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574515/real 1788574515] req@ffff8fc4ac820e00 x1875440516707200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574515.330566:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574515.330567:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574515.330569:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574515.330572:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574515.330574:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574515.330576:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574515.330580:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516707200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574515.330590:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516707200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330597:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574515.330599:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.330601:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574515.330603:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574515.330606:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516707200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330640:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574515.330646:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516707200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330654:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574515.330656:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.330659:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574515.330661:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.330663:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574515.330665:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574515.330669:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574515.330673:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574515.330675:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574515.330680:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516707200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574515.330689:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516707200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574515.330694:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574515.330698:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516707328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330705:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574515.330706:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574515.330708:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574515.330711:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574515.330713:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574515.330717:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516707328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574515.330720:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574515.330723:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574515.330731:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574515.330733:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574515.330737:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574515.330739:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574515.330744:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574515.330747:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574515.330752:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574515.330755:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516707328, portal 10 00000100:00000001:0.0:1788574515.330757:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574515.330759:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788574515.330766:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516707328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330775:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574515.330777:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574515.330780:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574515.330783:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516707328, offset 0 00000400:00000200:0.0:1788574515.330789:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574515.330794:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574515.330798:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574515.330802:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574515.330805:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574515.330831:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574515.330835:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574515.330839:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547178:547178:256:4294967295] 192.168.202.128@tcp LPNI seq info [547101:547101:8:4294967295] 00000400:00000200:0.0:1788574515.330844:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574515.330851:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574515.330856:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574515.330861:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574515.330866:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574515.330869:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574515.330881:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574515.330889:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574515.330893:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574515.330897:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574515.330902:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574515.330904:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574515.330907:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574515.330911:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516707328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330921:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574515.330924:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516707328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.330931:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.330933:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574515.330937:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574515.330941:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574515.330945:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574515.330947:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309875227 00000100:00000001:0.0:1788574515.330950:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309879022 00000100:00000001:0.0:1788574515.330952:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309875227 00000100:00000001:0.0:1788574515.330955:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309879022 00000100:00000001:0.0:1788574515.330957:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574515.330959:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.330962:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574515.330966:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574515/real 1788574515] req@ffff8fc4ac823800 x1875440516707328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574515.330975:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574515.330977:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574515.330978:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574515.330981:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574515.330983:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574515.330992:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574515.330997:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516707328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574515.331007:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516707328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331014:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574515.331016:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331018:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574515.331020:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574515.331024:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516707328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331033:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574515.331037:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516707328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331045:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574515.331048:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331049:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574515.331051:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331053:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574515.331056:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574515.331059:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574515.331064:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574515.331066:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574515.331070:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516707328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574515.331080:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516707328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574515.331085:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574515.331097:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516707456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331105:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574515.331107:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574515.331110:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574515.331112:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574515.331115:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574515.331119:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516707456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574515.331123:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574515.331125:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574515.331127:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574515.331129:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574515.331133:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574515.331136:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574515.331141:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574515.331144:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574515.331149:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574515.331152:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516707456, portal 4 00000100:00000001:0.0:1788574515.331154:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574515.331156:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788574515.331162:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516707456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331170:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574515.331172:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574515.331175:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574515.331178:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516707456, offset 0 00000400:00000200:0.0:1788574515.331182:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574515.331187:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574515.331191:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574515.331194:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574515.331198:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574515.331201:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574515.331203:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574515.331214:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547179:547179:256:4294967295] 192.168.202.128@tcp LPNI seq info [547102:547102:8:4294967295] 00000400:00000200:0.0:1788574515.331218:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574515.331226:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574515.331232:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574515.331236:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574515.331241:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574515.331244:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574515.331256:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574515.331258:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574515.331261:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574515.331265:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574515.331268:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574515.331270:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574515.331273:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574515.331277:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516707456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331287:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574515.331290:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516707456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331298:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.331299:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574515.331303:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574515.331307:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574515.331311:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574515.331313:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309875367 00000100:00000001:0.0:1788574515.331316:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309879022 00000100:00000001:0.0:1788574515.331319:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309875367 00000100:00000001:0.0:1788574515.331322:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309879022 00000100:00000001:0.0:1788574515.331324:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574515.331333:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.331335:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574515.331341:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574515/real 1788574515] req@ffff8fc4ac820a80 x1875440516707456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574515.331352:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574515.331354:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574515.331356:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574515.331359:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574515.331361:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574515.331363:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574515.331367:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516707456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574515.331376:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516707456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331384:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574515.331385:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331387:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574515.331389:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574515.331393:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516707456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331400:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574515.331404:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516707456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331412:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574515.331415:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331417:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574515.331419:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331421:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574515.331423:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574515.331427:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574515.331437:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574515.331440:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574515.331446:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516707456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574515.331455:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516707456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574515.331460:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574515.331464:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516707584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331472:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574515.331474:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574515.331476:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574515.331478:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574515.331480:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574515.331484:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516707584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574515.331487:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574515.331490:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574515.331491:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574515.331493:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574515.331497:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574515.331499:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574515.331503:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574515.331506:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574515.331510:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574515.331514:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516707584, portal 4 00000100:00000001:0.0:1788574515.331516:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574515.331518:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788574515.331524:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516707584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331532:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574515.331541:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574515.331544:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574515.331548:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516707584, offset 0 00000400:00000200:0.0:1788574515.331553:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574515.331558:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574515.331563:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574515.331566:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574515.331569:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574515.331572:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574515.331575:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574515.331579:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547180:547180:256:4294967295] 192.168.202.128@tcp LPNI seq info [547103:547103:8:4294967295] 00000400:00000200:0.0:1788574515.331583:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574515.331590:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574515.331596:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574515.331599:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574515.331604:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574515.331607:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574515.331636:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574515.331638:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574515.331642:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574515.331645:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574515.331649:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574515.331651:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574515.331654:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574515.331659:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516707584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331668:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574515.331671:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516707584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331678:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.331681:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574515.331692:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574515.331696:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574515.331700:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574515.331703:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309875369 00000100:00000001:0.0:1788574515.331706:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309879022 00000100:00000001:0.0:1788574515.331708:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309875369 00000100:00000001:0.0:1788574515.331711:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309879022 00000100:00000001:0.0:1788574515.331713:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574515.331716:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.331718:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574515.331723:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574515/real 1788574515] req@ffff8fc4ac821c00 x1875440516707584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574515.331732:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574515.331734:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574515.331736:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574515.331739:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574515.331740:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574515.331743:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574515.331746:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516707584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574515.331756:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516707584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331763:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574515.331765:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331767:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574515.331768:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574515.331772:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516707584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331780:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574515.331784:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516707584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331799:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574515.331802:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331804:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574515.331805:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331829:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574515.331832:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574515.331835:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574515.331840:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574515.331842:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574515.331848:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516707584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574515.331857:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516707584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574515.331861:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331863:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574515.331866:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516707200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331874:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574515.331876:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574515.331878:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574515.331881:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574515.331883:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574515.331885:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574515.331887:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574515.331888:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788574515.331892:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574515.331893:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574515.331895:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788574515.331898:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574515.331900:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331910:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574515.331915:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516707328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331923:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574515.331925:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574515.331927:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574515.331930:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574515.331931:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574515.331933:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574515.331935:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574515.331936:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788574515.331939:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574515.331940:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574515.331942:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788574515.331945:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574515.331946:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331948:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574515.331952:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516707456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.331960:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574515.331961:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574515.331963:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574515.331966:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574515.331967:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574515.331969:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574515.331971:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574515.331972:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788574515.331976:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574515.331977:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574515.331979:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788574515.331981:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574515.331983:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.331985:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574515.331989:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516707584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574515.332010:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574515.332012:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574515.332013:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574515.332016:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574515.332017:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574515.332019:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574515.332021:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574515.332022:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788574515.332025:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574515.332026:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574515.332028:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788574515.332031:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574515.332032:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.332034:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.332038:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574515.332040:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.332042:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574515.332044:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574515.840927:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574515.840933:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574515.840937:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574515.840945:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574515.840953:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574515.840961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.840982:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574515.840986:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574515.840990:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574516.545023:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574516.545028:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574516.545032:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574517.888967:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574517.888974:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574517.888978:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574517.888986:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574517.888991:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574517.888998:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574517.889027:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574517.889033:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574517.889038:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574518.593091:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574518.593107:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574518.593110:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574518.912988:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574518.912997:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574518.913005:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574518.913015:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574518.913027:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574518.913039:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574518.913072:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574518.913079:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574518.913087:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574519.168903:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309879022, at 4309879040 00000100:00000010:0.0:1788574519.168915:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574519.617065:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574519.617069:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574519.617071:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574519.936972:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574519.936980:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574519.936989:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574519.937016:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574519.937039:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574519.937048:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574519.937061:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574519.937064:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574519.937075:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.449164:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574520.449174:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574520.449180:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574520.449200:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516707712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.449234:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574520.449240:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574520.449266:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574520.449272:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574520.449279:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574520.449289:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516707712:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574520.449302:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574520.449313:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574520.449320:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574520.449325:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574520.449335:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574520.449340:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574520.449354:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574520.449364:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574520.449375:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574520.449389:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516707712, portal 25 00000100:00000001:0.0:1788574520.449397:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574520.449403:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788574520.449421:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516707712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.449438:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574520.449450:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574520.449458:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574520.449465:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516707712, offset 0 00000400:00000200:0.0:1788574520.449485:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574520.449504:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574520.449521:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574520.449537:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574520.449543:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574520.449550:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574520.449554:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574520.449566:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547181:547181:256:4294967295] 192.168.202.128@tcp LPNI seq info [547104:547104:8:4294967295] 00000400:00000200:0.0:1788574520.449591:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574520.449620:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574520.449640:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574520.449664:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574520.449692:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574520.449701:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788574520.449724:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574520.449727:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574520.449739:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574520.449751:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574520.449758:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574520.449766:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574520.449780:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574520.449790:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516707712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.449870:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574520.449880:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516707712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.449896:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.449909:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574520.449919:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574520.449933:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574520.449942:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309884142 00000100:00000010:0.0:1788574520.449955:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574520.449966:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574520.449978:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309884142 00000100:00000010:0.0:1788574520.449987:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b507e0. 00000100:00000001:0.0:1788574520.449994:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574520.450000:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574520.450005:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.450008:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574520.450016:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574520/real 1788574520] req@ffff8fc4ac820700 x1875440516707712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574520.450127:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574520.450133:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574520.450141:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574520.450150:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574520.450153:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574520.450157:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574520.450166:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516707712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574520.450190:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516707712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450235:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574520.450242:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.450250:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574520.450255:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574520.450271:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516707712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450305:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574520.450326:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516707712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450359:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574520.450372:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.450383:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574520.450386:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.450394:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574520.450403:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574520.450408:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574520.450419:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574520.450422:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574520.450432:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516707712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574520.450465:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516707712:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574520.450480:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.450492:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574520.450504:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516707712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450525:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574520.450530:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574520.450533:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574520.450536:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574520.450539:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574520.450540:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574520.450542:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574520.450545:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788574520.450548:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574520.450551:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574520.450552:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788574520.450556:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574520.450557:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.450559:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.450566:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574520.450567:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.450570:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574520.450571:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574520.450573:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574520.450577:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516707840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450583:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574520.450585:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574520.450587:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574520.450589:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574520.450591:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574520.450593:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516707840:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574520.450602:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574520.450605:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574520.450606:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574520.450608:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574520.450610:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574520.450611:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574520.450615:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574520.450618:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574520.450621:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574520.450624:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516707840, portal 10 00000100:00000001:0.0:1788574520.450625:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574520.450626:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788574520.450632:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516707840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450637:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574520.450638:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574520.450640:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574520.450641:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516707840, offset 0 00000400:00000200:0.0:1788574520.450662:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574520.450666:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574520.450669:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574520.450671:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574520.450673:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574520.450675:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574520.450677:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574520.450679:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547182:547182:256:4294967295] 192.168.202.128@tcp LPNI seq info [547105:547105:8:4294967295] 00000400:00000200:0.0:1788574520.450683:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574520.450687:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574520.450691:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574520.450693:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574520.450697:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574520.450703:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788574520.450712:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574520.450714:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574520.450716:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574520.450719:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574520.450722:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574520.450723:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574520.450725:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574520.450728:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516707840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450734:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574520.450736:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516707840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450741:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.450742:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574520.450744:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574520.450748:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574520.450750:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574520.450752:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309880339 00000100:00000001:0.0:1788574520.450754:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309884142 00000100:00000001:0.0:1788574520.450756:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309880339 00000100:00000001:0.0:1788574520.450757:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309884142 00000100:00000001:0.0:1788574520.450759:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574520.450760:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.450761:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574520.450764:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574520/real 1788574520] req@ffff8fc4ac822300 x1875440516707840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574520.450769:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574520.450770:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574520.450774:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574520.450776:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574520.450777:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574520.450780:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574520.450782:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516707840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574520.450791:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516707840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450798:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574520.450800:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.450802:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574520.450804:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574520.450831:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516707840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450840:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574520.450845:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516707840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450852:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574520.450855:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.450857:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574520.450858:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.450860:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574520.450863:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574520.450866:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574520.450870:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574520.450873:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574520.450877:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516707840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574520.450886:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516707840:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574520.450897:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574520.450903:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516707968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450911:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574520.450912:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574520.450915:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574520.450917:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574520.450920:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574520.450923:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516707968:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574520.450927:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574520.450929:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574520.450931:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574520.450933:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574520.450937:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574520.450939:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574520.450943:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574520.450946:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574520.450950:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574520.450953:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516707968, portal 10 00000100:00000001:0.0:1788574520.450956:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574520.450957:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788574520.450963:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516707968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.450971:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574520.450973:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574520.450976:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574520.450978:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516707968, offset 0 00000400:00000200:0.0:1788574520.450983:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574520.450988:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574520.450991:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574520.450994:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574520.451004:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574520.451006:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574520.451009:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574520.451012:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547183:547183:256:4294967295] 192.168.202.128@tcp LPNI seq info [547106:547106:8:4294967295] 00000400:00000200:0.0:1788574520.451017:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574520.451025:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574520.451030:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574520.451034:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574520.451038:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574520.451041:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788574520.451058:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574520.451061:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574520.451064:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574520.451068:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574520.451071:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574520.451073:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574520.451076:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574520.451080:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516707968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451089:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574520.451092:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516707968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451100:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.451101:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574520.451105:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574520.451109:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574520.451113:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574520.451115:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309880347 00000100:00000001:0.0:1788574520.451118:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309884142 00000100:00000001:0.0:1788574520.451120:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309880347 00000100:00000001:0.0:1788574520.451129:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309884142 00000100:00000001:0.0:1788574520.451131:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574520.451133:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.451136:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574520.451141:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574520/real 1788574520] req@ffff8fc4ac823480 x1875440516707968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574520.451151:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574520.451152:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574520.451154:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574520.451157:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574520.451159:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574520.451161:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574520.451164:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516707968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574520.451174:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516707968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451181:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574520.451182:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.451185:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574520.451186:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574520.451190:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516707968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451197:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574520.451202:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516707968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451208:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574520.451211:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.451212:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574520.451213:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.451222:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574520.451226:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574520.451229:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574520.451233:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574520.451236:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574520.451240:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516707968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574520.451249:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516707968:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574520.451254:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574520.451258:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516708096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451265:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574520.451267:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574520.451269:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574520.451272:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574520.451274:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574520.451277:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516708096:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574520.451281:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574520.451283:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574520.451285:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574520.451286:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574520.451290:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574520.451292:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574520.451296:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574520.451299:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574520.451304:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574520.451307:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516708096, portal 4 00000100:00000001:0.0:1788574520.451309:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574520.451310:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788574520.451317:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516708096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451330:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574520.451334:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574520.451336:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574520.451339:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516708096, offset 0 00000400:00000200:0.0:1788574520.451343:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574520.451348:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574520.451352:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574520.451355:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574520.451358:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574520.451360:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574520.451363:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574520.451366:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547184:547184:256:4294967295] 192.168.202.128@tcp LPNI seq info [547107:547107:8:4294967295] 00000400:00000200:0.0:1788574520.451370:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574520.451377:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574520.451382:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574520.451386:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574520.451390:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574520.451394:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788574520.451404:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574520.451406:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574520.451410:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574520.451414:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574520.451417:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574520.451419:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574520.451422:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574520.451426:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516708096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451434:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574520.451438:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516708096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451456:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.451458:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574520.451463:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574520.451466:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574520.451470:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574520.451473:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309880487 00000100:00000001:0.0:1788574520.451475:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309884142 00000100:00000001:0.0:1788574520.451478:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309880487 00000100:00000001:0.0:1788574520.451480:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309884142 00000100:00000001:0.0:1788574520.451483:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574520.451485:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.451487:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574520.451491:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574520/real 1788574520] req@ffff8fc4ac821f80 x1875440516708096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574520.451500:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574520.451501:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574520.451503:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574520.451505:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574520.451507:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574520.451510:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574520.451513:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516708096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574520.451522:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516708096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451528:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574520.451530:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.451532:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574520.451534:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574520.451538:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516708096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451552:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574520.451557:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516708096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451565:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574520.451568:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.451570:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574520.451571:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.451573:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574520.451575:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574520.451578:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574520.451582:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574520.451585:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574520.451589:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516708096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574520.451598:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516708096:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574520.451602:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574520.451607:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516708224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451614:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574520.451615:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574520.451618:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574520.451620:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574520.451622:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574520.451625:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516708224:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574520.451629:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574520.451631:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574520.451639:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574520.451641:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574520.451658:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574520.451661:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574520.451665:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574520.451668:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574520.451672:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574520.451675:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516708224, portal 4 00000100:00000001:0.0:1788574520.451677:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574520.451679:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788574520.451686:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516708224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451694:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574520.451696:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574520.451698:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574520.451701:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516708224, offset 0 00000400:00000200:0.0:1788574520.451705:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574520.451710:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574520.451714:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574520.451717:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574520.451720:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574520.451722:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574520.451725:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574520.451728:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547185:547185:256:4294967295] 192.168.202.128@tcp LPNI seq info [547108:547108:8:4294967295] 00000400:00000200:0.0:1788574520.451732:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574520.451739:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574520.451744:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574520.451747:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574520.451751:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574520.451754:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788574520.451766:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574520.451774:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574520.451777:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574520.451781:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574520.451786:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574520.451788:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574520.451791:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574520.451795:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516708224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451803:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574520.451826:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516708224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451834:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.451836:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574520.451840:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574520.451844:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574520.451848:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574520.451851:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309880490 00000100:00000001:0.0:1788574520.451854:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309884142 00000100:00000001:0.0:1788574520.451856:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309880490 00000100:00000001:0.0:1788574520.451858:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309884142 00000100:00000001:0.0:1788574520.451860:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574520.451863:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.451865:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574520.451870:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574520/real 1788574520] req@ffff8fc4ac822d80 x1875440516708224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574520.451879:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574520.451880:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574520.451882:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574520.451885:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574520.451887:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574520.451896:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574520.451900:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516708224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574520.451909:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516708224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451916:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574520.451918:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.451920:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574520.451922:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574520.451925:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516708224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451932:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574520.451935:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516708224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451941:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574520.451944:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.451945:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574520.451946:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.451948:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574520.451951:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574520.451953:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574520.451957:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574520.451959:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574520.451963:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516708224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574520.451972:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516708224:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574520.451976:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.451978:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574520.451989:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516707840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.451998:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574520.452000:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574520.452002:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574520.452005:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574520.452007:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574520.452008:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574520.452010:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574520.452012:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788574520.452015:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574520.452017:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574520.452019:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788574520.452022:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574520.452024:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.452026:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574520.452029:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516707968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.452036:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574520.452038:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574520.452039:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574520.452041:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574520.452043:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574520.452044:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574520.452046:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574520.452048:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574520.452050:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574520.452051:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574520.452053:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788574520.452055:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574520.452057:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.452059:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574520.452062:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516708096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.452069:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574520.452076:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574520.452078:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574520.452082:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574520.452083:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574520.452085:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574520.452087:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574520.452088:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788574520.452091:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574520.452092:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574520.452094:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788574520.452096:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574520.452098:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.452100:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574520.452103:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516708224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574520.452110:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574520.452112:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574520.452113:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574520.452115:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574520.452117:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574520.452118:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574520.452120:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574520.452122:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788574520.452124:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574520.452125:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574520.452127:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788574520.452129:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574520.452131:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.452133:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.452138:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574520.452140:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.452142:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574520.452144:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574520.960954:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574520.960970:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574520.960976:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574520.960986:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574520.961024:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574520.961035:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.961058:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574520.961061:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574520.961063:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574521.665014:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574521.665018:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574521.665020:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574521.984895:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574521.984900:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574521.984903:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574521.984910:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574521.984914:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574521.984920:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574521.984935:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574521.984939:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574521.984943:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574523.008912:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574523.008919:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574523.008924:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574523.008934:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574523.008942:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574523.008949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574523.008956:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574523.008959:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574523.008965:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574523.713010:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574523.713014:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574523.713023:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574524.033047:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574524.033051:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574524.033052:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574524.033055:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574524.033057:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574524.033060:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574524.033061:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574524.033063:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574524.033065:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574524.033101:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574524.033105:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574524.033107:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574524.033112:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574524.033115:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574524.033117:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574524.033118:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574524.033119:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574524.033121:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574524.288876:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309884142, at 4309884160 00000100:00000010:0.0:1788574524.288893:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574524.736991:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574524.736996:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574524.736999:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574525.569117:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574525.569123:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574525.569129:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574525.569141:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516708352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.569167:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574525.569173:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574525.569179:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574525.569185:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574525.569192:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574525.569201:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516708352:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574525.569211:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574525.569217:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574525.569222:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574525.569229:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574525.569237:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574525.569244:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574525.569263:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574525.569273:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574525.569285:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574525.569296:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516708352, portal 25 00000100:00000001:0.0:1788574525.569310:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574525.569314:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788574525.569328:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516708352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.569345:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574525.569354:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574525.569359:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574525.569365:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516708352, offset 0 00000400:00000200:0.0:1788574525.569379:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574525.569391:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574525.569412:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574525.569423:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574525.569432:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574525.569440:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574525.569449:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574525.569458:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547186:547186:256:4294967295] 192.168.202.128@tcp LPNI seq info [547109:547109:8:4294967295] 00000400:00000200:0.0:1788574525.569468:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574525.569487:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574525.569501:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574525.569511:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574525.569523:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574525.569531:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788574525.569564:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574525.569569:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574525.569579:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574525.569592:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574525.569602:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574525.569608:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574525.569617:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574525.569627:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516708352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.569672:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574525.569696:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516708352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.569713:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574525.569717:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574525.569726:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574525.569733:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574525.569738:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309889261 00000100:00000010:0.0:1788574525.569745:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574525.569754:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574525.569763:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309889261 00000100:00000010:0.0:1788574525.569768:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50ae0. 00000100:00000001:0.0:1788574525.569776:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574525.569782:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574525.569788:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574525.569794:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574525.569803:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574525/real 1788574525] req@ffff8fc4ac822a00 x1875440516708352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574525.569845:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574525.569847:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574525.569849:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574525.569852:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574525.569854:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574525.569857:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574525.569862:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516708352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574525.569871:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516708352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.569879:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574525.569886:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.569889:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574525.569891:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574525.569895:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516708352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.569903:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574525.569908:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516708352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.569917:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574525.569920:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.569922:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574525.569924:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.569926:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574525.569928:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574525.569931:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574525.569935:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574525.569938:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574525.569942:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516708352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574525.569951:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516708352:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574525.569956:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.569958:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574525.569961:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516708352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.569969:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574525.569971:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574525.569974:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574525.569977:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574525.569979:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574525.569987:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574525.569990:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574525.569992:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788574525.569996:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574525.569998:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574525.570000:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788574525.570004:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574525.570005:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.570008:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.570015:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574525.570017:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574525.570019:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574525.570021:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574525.570022:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574525.570027:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516708480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570034:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574525.570036:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574525.570038:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574525.570040:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574525.570042:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574525.570045:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516708480:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574525.570048:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574525.570050:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574525.570052:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574525.570054:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574525.570057:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574525.570059:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574525.570064:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574525.570067:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574525.570071:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574525.570074:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516708480, portal 10 00000100:00000001:0.0:1788574525.570076:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574525.570078:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788574525.570085:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516708480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570097:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574525.570100:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574525.570103:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574525.570106:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516708480, offset 0 00000400:00000200:0.0:1788574525.570111:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574525.570115:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574525.570120:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574525.570123:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574525.570126:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574525.570129:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574525.570131:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574525.570134:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547187:547187:256:4294967295] 192.168.202.128@tcp LPNI seq info [547110:547110:8:4294967295] 00000400:00000200:0.0:1788574525.570139:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574525.570145:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574525.570150:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574525.570154:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574525.570158:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574525.570162:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788574525.570173:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574525.570175:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574525.570178:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574525.570182:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574525.570185:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574525.570187:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574525.570190:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574525.570194:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516708480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570203:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574525.570206:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516708480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570225:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574525.570226:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574525.570231:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574525.570235:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574525.570239:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574525.570242:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309885459 00000100:00000001:0.0:1788574525.570244:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309889261 00000100:00000001:0.0:1788574525.570247:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309885459 00000100:00000001:0.0:1788574525.570249:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309889261 00000100:00000001:0.0:1788574525.570252:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574525.570254:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574525.570256:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574525.570261:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574525/real 1788574525] req@ffff8fc4ac823100 x1875440516708480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574525.570270:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574525.570271:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574525.570273:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574525.570276:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574525.570278:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574525.570280:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574525.570284:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516708480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574525.570293:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516708480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570300:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574525.570301:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.570303:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574525.570305:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574525.570310:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516708480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570323:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574525.570328:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516708480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570335:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574525.570338:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.570340:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574525.570341:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.570343:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574525.570345:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574525.570348:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574525.570352:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574525.570354:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574525.570359:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516708480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574525.570367:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516708480:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574525.570372:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574525.570375:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516708608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570382:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574525.570384:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574525.570386:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574525.570388:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574525.570390:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574525.570394:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516708608:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574525.570397:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574525.570399:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574525.570406:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574525.570409:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574525.570412:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574525.570414:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574525.570418:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574525.570421:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574525.570424:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574525.570427:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516708608, portal 10 00000100:00000001:0.0:1788574525.570430:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574525.570431:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788574525.570438:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516708608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570445:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574525.570448:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574525.570450:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574525.570452:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516708608, offset 0 00000400:00000200:0.0:1788574525.570457:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574525.570461:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574525.570465:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574525.570468:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574525.570471:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574525.570473:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574525.570476:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574525.570480:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547188:547188:256:4294967295] 192.168.202.128@tcp LPNI seq info [547111:547111:8:4294967295] 00000400:00000200:0.0:1788574525.570484:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574525.570491:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574525.570496:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574525.570500:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574525.570504:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574525.570507:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788574525.570519:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574525.570526:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574525.570529:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574525.570533:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574525.570537:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574525.570539:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574525.570542:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574525.570546:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516708608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570554:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574525.570557:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516708608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570564:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574525.570566:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574525.570570:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574525.570573:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574525.570577:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574525.570579:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309885466 00000100:00000001:0.0:1788574525.570582:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309889261 00000100:00000001:0.0:1788574525.570584:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309885466 00000100:00000001:0.0:1788574525.570586:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309889261 00000100:00000001:0.0:1788574525.570589:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574525.570591:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574525.570593:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574525.570598:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574525/real 1788574525] req@ffff8fc4ac821500 x1875440516708608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574525.570606:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574525.570607:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574525.570609:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574525.570612:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574525.570613:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574525.570622:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574525.570626:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516708608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574525.570636:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516708608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570643:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574525.570645:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.570647:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574525.570648:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574525.570652:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516708608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570659:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574525.570663:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516708608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570671:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574525.570690:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.570691:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574525.570692:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.570695:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574525.570697:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574525.570701:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574525.570705:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574525.570707:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574525.570712:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516708608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574525.570721:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516708608:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574525.570725:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574525.570737:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516708736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570744:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574525.570746:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574525.570748:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574525.570750:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574525.570752:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574525.570756:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516708736:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574525.570759:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574525.570761:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574525.570763:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574525.570765:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574525.570768:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574525.570771:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574525.570775:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574525.570778:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574525.570782:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574525.570785:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516708736, portal 4 00000100:00000001:0.0:1788574525.570787:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574525.570788:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788574525.570795:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516708736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570802:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574525.570805:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574525.570826:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574525.570830:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516708736, offset 0 00000400:00000200:0.0:1788574525.570835:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574525.570840:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574525.570843:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574525.570846:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574525.570849:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574525.570851:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574525.570854:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574525.570863:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547189:547189:256:4294967295] 192.168.202.128@tcp LPNI seq info [547112:547112:8:4294967295] 00000400:00000200:0.0:1788574525.570867:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574525.570875:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574525.570880:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574525.570884:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574525.570888:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574525.570891:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788574525.570903:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574525.570905:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574525.570908:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574525.570912:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574525.570916:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574525.570917:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574525.570920:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574525.570924:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516708736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570933:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574525.570937:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516708736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.570944:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574525.570946:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574525.570949:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574525.570952:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574525.570955:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574525.570958:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309885607 00000100:00000001:0.0:1788574525.570961:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309889261 00000100:00000001:0.0:1788574525.570963:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309885607 00000100:00000001:0.0:1788574525.570965:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309889261 00000100:00000001:0.0:1788574525.570967:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574525.570976:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574525.570978:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574525.570983:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574525/real 1788574525] req@ffff8fc4ac823b80 x1875440516708736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574525.570992:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574525.570994:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574525.570996:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574525.570999:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574525.571000:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574525.571003:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574525.571006:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516708736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574525.571016:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516708736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571023:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574525.571025:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.571027:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574525.571028:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574525.571032:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516708736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571039:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574525.571044:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516708736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571051:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574525.571054:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.571055:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574525.571057:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.571059:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574525.571061:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574525.571064:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574525.571075:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574525.571077:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574525.571082:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516708736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574525.571091:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516708736:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574525.571095:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574525.571099:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516708864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571106:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574525.571108:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574525.571111:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574525.571113:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574525.571115:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574525.571118:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516708864:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574525.571121:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574525.571123:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574525.571125:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574525.571127:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574525.571131:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574525.571133:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574525.571137:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574525.571140:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574525.571144:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574525.571147:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516708864, portal 4 00000100:00000001:0.0:1788574525.571149:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574525.571151:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788574525.571156:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516708864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571164:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574525.571172:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574525.571174:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574525.571178:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516708864, offset 0 00000400:00000200:0.0:1788574525.571182:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574525.571187:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574525.571190:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574525.571194:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574525.571196:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574525.571198:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574525.571201:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574525.571204:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547190:547190:256:4294967295] 192.168.202.128@tcp LPNI seq info [547113:547113:8:4294967295] 00000400:00000200:0.0:1788574525.571209:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574525.571216:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574525.571221:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574525.571224:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574525.571228:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574525.571231:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788574525.571243:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574525.571245:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574525.571248:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574525.571252:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574525.571255:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574525.571257:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574525.571260:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574525.571264:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516708864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571272:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574525.571275:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516708864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571282:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574525.571284:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574525.571295:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574525.571298:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574525.571303:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574525.571305:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309885609 00000100:00000001:0.0:1788574525.571307:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309889261 00000100:00000001:0.0:1788574525.571309:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309885609 00000100:00000001:0.0:1788574525.571312:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309889261 00000100:00000001:0.0:1788574525.571314:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574525.571316:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574525.571318:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574525.571323:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574525/real 1788574525] req@ffff8fc4ac820000 x1875440516708864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574525.571332:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574525.571333:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574525.571335:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574525.571338:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574525.571340:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574525.571342:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574525.571346:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516708864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574525.571355:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516708864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571362:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574525.571363:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.571365:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574525.571367:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574525.571371:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516708864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571378:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574525.571382:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516708864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571395:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574525.571398:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.571399:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574525.571401:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.571403:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574525.571405:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574525.571408:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574525.571412:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574525.571414:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574525.571419:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516708864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574525.571428:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516708864:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574525.571432:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.571434:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574525.571437:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516708480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571445:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574525.571447:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574525.571449:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574525.571452:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574525.571453:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574525.571455:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574525.571457:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574525.571459:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788574525.571462:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574525.571464:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574525.571465:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788574525.571468:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574525.571470:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.571479:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574525.571484:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516708608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571491:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574525.571493:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574525.571495:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574525.571497:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574525.571499:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574525.571500:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574525.571502:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574525.571504:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862ac00. 02000000:00000001:0.0:1788574525.571506:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574525.571507:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574525.571509:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788574525.571511:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574525.571513:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.571515:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574525.571518:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516708736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571525:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574525.571527:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574525.571528:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574525.571531:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574525.571532:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574525.571533:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574525.571535:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574525.571537:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788574525.571539:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574525.571540:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574525.571542:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788574525.571544:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574525.571545:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.571547:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574525.571550:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516708864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574525.571569:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574525.571570:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574525.571572:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574525.571574:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574525.571576:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574525.571577:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574525.571579:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574525.571581:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788574525.571583:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574525.571584:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574525.571586:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788574525.571588:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574525.571589:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.571592:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574525.571596:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574525.571597:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574525.571599:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574525.571601:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574526.080955:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574526.080961:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574526.080965:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574526.080969:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574526.080975:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574526.080978:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574526.080988:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574526.080991:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574526.080994:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574526.784961:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574526.784965:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574526.784968:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574527.104935:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574527.104940:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574527.104943:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574527.104950:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574527.104954:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574527.104958:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574527.104978:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574527.104981:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574527.104985:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574528.128937:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574528.128941:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574528.128944:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574528.128948:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574528.128953:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574528.128957:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574528.128968:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574528.128973:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574528.128978:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574528.832986:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574528.832991:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574528.832995:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574529.153062:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574529.153072:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574529.153077:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574529.153082:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574529.153091:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574529.153098:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574529.153105:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574529.153108:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574529.153112:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574529.408954:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309889261, at 4309889280 00000100:00000010:0.0:1788574529.408974:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574529.857057:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574529.857062:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574529.857064:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574530.176919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574530.176930:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574530.176933:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574530.176943:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574530.176951:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574530.176960:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574530.176974:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574530.176977:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574530.176995:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574530.688907:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574530.688916:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574530.688919:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574530.688922:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574530.688926:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574530.688931:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15218 00000400:00000200:0.0:1788574530.688939:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574530.688945:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574530.688947:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574530.688949:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.688952:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.688954:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.688956:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574530.688959:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574530.688960:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574530.688962:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574530.688965:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574530.688966:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574530.688968:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574530.688970:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.688972:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.688973:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.688974:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574530.688976:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574530.688978:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574530.688980:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574530.688982:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574530.688984:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574530.688986:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689004:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574530.689005:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574530.689011:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574530.689013:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574530.689016:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574530.689018:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689020:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574530.689022:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574530.689026:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574530.689031:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574530.689037:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574530.689044:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516708992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574530.689052:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574530.689055:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574530.689058:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516708992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574530.689076:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574530.689079:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574530.689081:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574530.689082:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574530.689084:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574530.689085:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574530.689088:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15218 00000400:00000200:0.0:1788574530.689091:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574530.689099:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574530.689101:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574530.689106:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689108:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.689112:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689116:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574530.689126:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574530.689129:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574530.689132:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574530.689135:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574530.689138:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574530.689140:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574530.689143:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689145:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.689146:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689147:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574530.689148:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574530.689150:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689151:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574530.689155:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574530.689160:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574530.689164:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689168:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574530.689171:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574530.689176:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689181:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574530.689186:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574530.689190:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574530.689194:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574530.689197:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689199:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574530.689200:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574530.689203:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574530.689209:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574530.689214:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574530.689222:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516709120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574530.689235:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574530.689243:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574530.689250:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516709120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574530.689265:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574530.689269:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574530.689277:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574530.689280:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574530.689284:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574530.689288:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574530.689293:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15218 00000400:00000200:0.0:1788574530.689298:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574530.689304:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574530.689309:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574530.689314:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689316:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.689320:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689323:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574530.689325:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574530.689328:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574530.689331:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574530.689334:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574530.689337:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574530.689342:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574530.689347:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689350:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.689355:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689359:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574530.689363:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574530.689370:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689379:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574530.689383:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574530.689387:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574530.689390:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689391:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574530.689392:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574530.689394:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689397:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574530.689404:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574530.689406:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574530.689409:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574530.689412:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689415:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574530.689415:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574530.689417:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574530.689421:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574530.689426:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574530.689433:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516709248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574530.689444:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574530.689449:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574530.689454:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516709248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574530.689466:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574530.689469:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574530.689473:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574530.689476:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574530.689479:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574530.689481:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574530.689484:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15218 00000400:00000200:0.0:1788574530.689492:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574530.689511:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574530.689515:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574530.689522:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689527:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.689530:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689535:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574530.689538:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574530.689539:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574530.689541:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574530.689544:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574530.689547:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574530.689550:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574530.689559:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689565:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.689570:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689575:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574530.689580:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574530.689588:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689594:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574530.689598:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574530.689604:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574530.689609:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689611:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574530.689618:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574530.689624:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689629:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574530.689636:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574530.689641:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574530.689645:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574530.689648:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689652:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574530.689676:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574530.689682:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574530.689691:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574530.689695:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574530.689719:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516709376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574530.689732:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574530.689736:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574530.689744:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516709376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574530.689763:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574530.689768:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574530.689778:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574530.689779:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574530.689781:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574530.689784:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574530.689788:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15218 00000400:00000200:0.0:1788574530.689798:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574530.689858:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574530.689863:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574530.689872:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689878:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.689882:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689887:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574530.689892:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574530.689893:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574530.689894:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574530.689897:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574530.689903:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574530.689908:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574530.689924:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689931:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574530.689938:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689943:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574530.689947:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574530.689951:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574530.689958:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574530.689963:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574530.689969:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574530.689976:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689981:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574530.689986:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574530.689993:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.689999:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574530.690006:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574530.690012:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574530.690017:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574530.690020:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574530.690025:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574530.690029:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574530.690034:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574530.690042:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574530.690049:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574530.690059:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516709504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574530.690075:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574530.690078:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574530.690087:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516709504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574530.690104:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574530.690112:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15228) 00000100:00000001:0.0:1788574531.201024:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574531.201037:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574531.201048:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574531.201053:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574531.201056:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574531.201058:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574531.201095:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574531.201101:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574531.201108:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574531.905131:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574531.905135:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574531.905138:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574532.224949:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574532.224965:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574532.224969:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574532.224984:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574532.225002:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574532.225013:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574532.225039:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574532.225044:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574532.225049:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574532.737202:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788574533.953075:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574533.953082:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574533.953086:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574534.273046:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574534.273057:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574534.273064:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574534.273070:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574534.273077:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574534.273087:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574534.273099:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574534.273102:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574534.273106:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574534.528936:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309894381, at 4309894400 00000100:00000010:0.0:1788574534.528952:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abde0. 00000020:00000001:0.0:1788574534.977041:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574534.977063:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574534.977067:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574536.321015:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574536.321020:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574536.321021:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574536.321024:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574536.321027:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574536.321029:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574536.321030:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574536.321032:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574536.321035:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574537.025012:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574537.025015:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574537.025018:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788574539.073101:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574539.073107:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574539.073111:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574539.392923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574539.392927:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574539.392931:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574539.392936:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574539.392941:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574539.392954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574539.392976:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574539.392980:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574539.392992:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574539.648912:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309899502, at 4309899520 00000100:00000010:0.0:1788574539.648926:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab480. 00000020:00000001:0.0:1788574540.097021:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574540.097025:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574540.097028:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574540.416975:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574540.416984:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574540.416989:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574540.416998:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574540.417003:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574540.417007:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574540.417038:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574540.417048:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574540.417059:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574540.928955:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574540.928967:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574540.928971:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574540.928979:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574540.928984:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574540.928992:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15228 00000400:00000200:0.0:1788574540.929003:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574540.929011:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574540.929014:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574540.929016:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929019:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574540.929021:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929023:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574540.929028:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574540.929029:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574540.929032:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574540.929035:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574540.929037:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574540.929039:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574540.929041:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929043:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574540.929044:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929045:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574540.929048:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574540.929050:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929052:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574540.929054:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574540.929057:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574540.929085:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.929088:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574540.929090:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574540.929096:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574540.929100:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574540.929103:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574540.929105:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.929108:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574540.929111:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574540.929116:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574540.929124:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574540.929131:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574540.929139:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516710272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574540.929148:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574540.929152:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574540.929156:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516710272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574540.929176:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574540.929179:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574540.929182:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574540.929183:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574540.929184:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574540.929186:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574540.929188:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15228 00000400:00000200:0.0:1788574540.929192:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574540.929194:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574540.929197:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574540.929199:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929201:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574540.929203:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929210:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574540.929214:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574540.929215:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574540.929217:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574540.929221:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574540.929225:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574540.929227:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574540.929233:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929236:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574540.929240:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929245:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574540.929250:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574540.929253:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929257:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574540.929259:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574540.929265:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574540.929268:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.929271:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574540.929277:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574540.929282:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.929285:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574540.929290:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574540.929294:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574540.929297:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574540.929298:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.929299:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574540.929301:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574540.929305:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574540.929314:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574540.929319:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574540.929326:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516710400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574540.929355:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574540.929359:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574540.929367:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516710400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574540.929379:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574540.929383:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574540.929385:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574540.929388:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574540.929390:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574540.929394:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574540.929399:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15228 00000400:00000200:0.0:1788574540.929405:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574540.929410:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574540.929413:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574540.929415:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929416:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574540.929419:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929424:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574540.929427:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574540.929428:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574540.929429:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574540.929432:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574540.929437:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574540.929439:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574540.929444:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929446:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574540.929451:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929454:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574540.929464:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574540.929470:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929476:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574540.929480:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574540.929482:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574540.929484:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.929486:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574540.929489:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574540.929494:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.929499:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574540.929502:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574540.929506:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574540.929509:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574540.929511:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.929516:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574540.929518:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574540.929519:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574540.929524:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574540.929529:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574540.929538:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516710528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574540.929549:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574540.929554:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574540.929561:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516710528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574540.929568:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574540.929572:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574540.929577:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574540.929580:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574540.929583:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574540.929589:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574540.929597:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15228 00000400:00000200:0.0:1788574540.929604:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574540.929607:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574540.929610:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574540.929618:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929623:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574540.929628:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929634:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574540.929639:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574540.929641:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574540.929645:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574540.929649:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574540.929656:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574540.929662:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574540.929671:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929676:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574540.929680:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929683:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574540.929688:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574540.929692:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929695:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574540.929701:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574540.929706:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574540.929712:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.929718:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574540.929722:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574540.929727:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.929730:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574540.929738:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574540.929741:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574540.929746:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574540.929755:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.929759:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574540.929777:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574540.929780:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574540.929786:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574540.929791:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574540.929799:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516710656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574540.929857:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574540.929860:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574540.929865:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516710656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574540.929878:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574540.929881:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574540.929885:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574540.929889:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574540.929891:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574540.929896:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574540.929900:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15228 00000400:00000200:0.0:1788574540.929908:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574540.929911:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574540.929913:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574540.929914:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929916:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574540.929921:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929926:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574540.929931:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574540.929934:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574540.929936:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574540.929939:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574540.929943:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574540.929952:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574540.929960:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929966:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574540.929969:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929973:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574540.929976:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574540.929979:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574540.929981:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574540.929982:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574540.929984:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574540.929990:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.929994:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574540.929997:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574540.930000:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.930006:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574540.930010:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574540.930014:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574540.930018:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574540.930021:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574540.930026:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574540.930028:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574540.930031:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574540.930035:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574540.930038:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574540.930046:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516710784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574540.930065:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574540.930069:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574540.930077:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516710784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574540.930089:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574540.930101:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15238) 00000100:00000001:0.0:1788574541.440948:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574541.440960:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574541.440966:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574541.440973:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574541.440978:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574541.440985:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574541.441001:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574541.441006:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574541.441012:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574542.145003:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574542.145006:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574542.145009:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574542.464921:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574542.464929:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574542.464932:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574542.464937:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574542.464944:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574542.464953:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574542.464960:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574542.464965:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574542.464974:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574542.465018:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574542.465024:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574542.465026:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574542.465042:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574542.465048:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574542.465051:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574542.465053:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574542.465055:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574542.465058:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574543.488939:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574543.488943:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574543.488945:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574543.488948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574543.488950:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574543.488952:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574543.488954:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574543.488974:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574543.488980:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574544.192995:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574544.193003:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574544.193008:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574544.512971:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574544.512982:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574544.512986:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574544.512994:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574544.513002:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574544.513007:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574544.513011:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574544.513020:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574544.513035:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574544.513102:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574544.513107:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574544.513110:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574544.513114:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574544.513119:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574544.513121:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574544.513123:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574544.513125:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574544.513127:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574544.768966:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309904621, at 4309904640 00000100:00000010:0.0:1788574544.768983:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab180. 00000020:00000001:0.0:1788574545.217023:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574545.217027:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574545.217029:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574545.537013:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574545.537023:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574545.537028:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574545.537032:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574545.537044:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574545.537050:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574545.537063:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574545.537065:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574545.537068:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.049233:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574546.049242:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574546.049249:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574546.049269:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516710912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.049317:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574546.049323:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574546.049335:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574546.049341:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574546.049352:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574546.049364:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516710912:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574546.049379:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574546.049386:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574546.049392:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574546.049406:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574546.049417:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574546.049423:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574546.049441:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574546.049449:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574546.049464:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574546.049477:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516710912, portal 25 00000100:00000001:0.0:1788574546.049485:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574546.049493:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788574546.049510:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516710912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.049563:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574546.049579:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574546.049591:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574546.049601:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516710912, offset 0 00000400:00000200:0.0:1788574546.049626:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574546.049646:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574546.049661:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574546.049671:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574546.049697:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574546.049707:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574546.049716:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574546.049735:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547206:547206:256:4294967295] 192.168.202.128@tcp LPNI seq info [547129:547129:8:4294967295] 00000400:00000200:0.0:1788574546.049758:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574546.049852:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574546.049876:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574546.049885:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574546.049899:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574546.049908:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574546.049960:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574546.049965:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574546.049979:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574546.049997:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574546.050009:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574546.050016:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574546.050025:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574546.050039:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516710912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050105:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574546.050118:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516710912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050166:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.050174:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574546.050185:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574546.050204:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574546.050214:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309909742 00000100:00000010:0.0:1788574546.050226:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574546.050237:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574546.050250:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309909742 00000100:00000010:0.0:1788574546.050269:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b504e0. 00000100:00000001:0.0:1788574546.050280:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574546.050294:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574546.050299:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.050306:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574546.050322:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574546/real 1788574546] req@ffff8fc4ac820000 x1875440516710912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574546.050371:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574546.050375:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574546.050380:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574546.050385:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574546.050389:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574546.050396:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574546.050403:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516710912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574546.050416:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516710912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050424:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574546.050426:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.050428:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574546.050431:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574546.050435:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516710912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050442:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574546.050447:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516710912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050454:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574546.050457:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.050459:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574546.050461:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.050472:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574546.050474:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574546.050478:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574546.050483:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574546.050486:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574546.050491:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516710912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574546.050500:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516710912:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574546.050504:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.050507:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574546.050510:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516710912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050517:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574546.050520:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574546.050523:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574546.050527:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574546.050529:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574546.050532:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574546.050534:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574546.050537:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788574546.050540:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574546.050542:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574546.050544:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788574546.050548:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574546.050549:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.050551:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.050558:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574546.050560:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.050563:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574546.050565:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574546.050566:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574546.050571:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516711040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050588:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574546.050590:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574546.050592:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574546.050595:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574546.050597:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574546.050601:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516711040:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574546.050604:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574546.050607:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574546.050609:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574546.050610:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574546.050613:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574546.050615:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574546.050620:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574546.050623:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574546.050627:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574546.050630:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516711040, portal 10 00000100:00000001:0.0:1788574546.050632:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574546.050634:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788574546.050641:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516711040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050647:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574546.050650:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574546.050652:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574546.050655:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516711040, offset 0 00000400:00000200:0.0:1788574546.050659:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574546.050663:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574546.050667:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574546.050670:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574546.050673:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574546.050676:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574546.050678:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574546.050686:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547207:547207:256:4294967295] 192.168.202.128@tcp LPNI seq info [547130:547130:8:4294967295] 00000400:00000200:0.0:1788574546.050691:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574546.050698:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574546.050703:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574546.050706:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574546.050710:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574546.050713:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574546.050726:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574546.050727:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574546.050730:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574546.050734:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574546.050737:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574546.050739:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574546.050742:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574546.050746:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516711040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050754:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574546.050757:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516711040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050764:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.050766:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574546.050768:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574546.050772:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574546.050775:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574546.050777:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309905939 00000100:00000001:0.0:1788574546.050780:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309909742 00000100:00000001:0.0:1788574546.050782:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309905939 00000100:00000001:0.0:1788574546.050785:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309909742 00000100:00000001:0.0:1788574546.050787:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574546.050826:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.050829:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574546.050834:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574546/real 1788574546] req@ffff8fc4ac823b80 x1875440516711040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574546.050843:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574546.050845:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574546.050847:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574546.050850:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574546.050851:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574546.050854:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574546.050857:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516711040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574546.050866:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516711040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050873:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574546.050874:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.050876:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574546.050878:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574546.050881:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516711040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050888:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574546.050892:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516711040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050899:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574546.050902:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.050903:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574546.050905:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.050907:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574546.050909:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574546.050912:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574546.050921:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574546.050923:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574546.050927:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516711040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574546.050936:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516711040:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574546.050940:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574546.050944:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516711168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.050951:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574546.050953:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574546.050955:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574546.050957:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574546.050959:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574546.050962:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516711168:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574546.050965:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574546.050968:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574546.050969:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574546.050971:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574546.050974:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574546.050977:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574546.050980:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574546.050983:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574546.050986:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574546.050989:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516711168, portal 10 00000100:00000001:0.0:1788574546.050991:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574546.050993:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788574546.050999:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516711168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051006:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574546.051012:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574546.051015:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574546.051017:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516711168, offset 0 00000400:00000200:0.0:1788574546.051021:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574546.051025:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574546.051029:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574546.051032:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574546.051035:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574546.051037:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574546.051040:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574546.051043:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547208:547208:256:4294967295] 192.168.202.128@tcp LPNI seq info [547131:547131:8:4294967295] 00000400:00000200:0.0:1788574546.051048:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574546.051054:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574546.051059:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574546.051062:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574546.051066:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574546.051069:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574546.051079:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574546.051081:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574546.051084:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574546.051088:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574546.051091:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574546.051092:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574546.051095:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574546.051099:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516711168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051106:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574546.051109:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516711168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051116:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.051122:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574546.051126:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574546.051129:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574546.051133:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574546.051135:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309905947 00000100:00000001:0.0:1788574546.051138:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309909742 00000100:00000001:0.0:1788574546.051140:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309905947 00000100:00000001:0.0:1788574546.051142:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309909742 00000100:00000001:0.0:1788574546.051145:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574546.051147:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.051148:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574546.051152:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574546/real 1788574546] req@ffff8fc4ac821500 x1875440516711168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574546.051160:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574546.051162:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574546.051164:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574546.051166:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574546.051168:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574546.051170:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574546.051173:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516711168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574546.051182:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516711168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051188:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574546.051190:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.051191:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574546.051193:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574546.051197:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516711168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051203:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574546.051212:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516711168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051220:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574546.051222:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.051225:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574546.051226:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.051228:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574546.051231:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574546.051234:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574546.051237:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574546.051240:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574546.051244:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516711168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574546.051252:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516711168:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574546.051256:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574546.051260:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516711296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051267:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574546.051268:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574546.051271:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574546.051273:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574546.051275:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574546.051277:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516711296:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574546.051281:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574546.051283:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574546.051285:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574546.051287:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574546.051290:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574546.051292:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574546.051300:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574546.051302:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574546.051305:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574546.051309:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516711296, portal 4 00000100:00000001:0.0:1788574546.051311:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574546.051313:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788574546.051318:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516711296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051325:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574546.051328:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574546.051330:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574546.051332:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516711296, offset 0 00000400:00000200:0.0:1788574546.051337:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574546.051340:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574546.051344:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574546.051347:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574546.051350:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574546.051352:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574546.051354:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574546.051357:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547209:547209:256:4294967295] 192.168.202.128@tcp LPNI seq info [547132:547132:8:4294967295] 00000400:00000200:0.0:1788574546.051362:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574546.051368:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574546.051372:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574546.051376:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574546.051380:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574546.051382:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574546.051393:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574546.051396:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574546.051398:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574546.051402:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574546.051405:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574546.051411:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574546.051415:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574546.051419:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516711296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051427:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574546.051430:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516711296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051436:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.051438:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574546.051441:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574546.051444:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574546.051447:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574546.051449:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309906087 00000100:00000001:0.0:1788574546.051452:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309909742 00000100:00000001:0.0:1788574546.051454:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309906087 00000100:00000001:0.0:1788574546.051457:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309909742 00000100:00000001:0.0:1788574546.051459:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574546.051461:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.051463:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574546.051467:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574546/real 1788574546] req@ffff8fc4ac823100 x1875440516711296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574546.051475:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574546.051476:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574546.051478:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574546.051481:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574546.051482:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574546.051484:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574546.051487:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516711296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574546.051496:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516711296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051507:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574546.051509:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.051511:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574546.051513:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574546.051517:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516711296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051524:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574546.051528:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516711296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051535:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574546.051537:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.051539:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574546.051540:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.051542:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574546.051544:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574546.051546:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574546.051550:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574546.051552:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574546.051556:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516711296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574546.051564:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516711296:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574546.051568:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574546.051572:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516711424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051579:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574546.051580:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574546.051586:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574546.051588:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574546.051590:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574546.051594:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516711424:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574546.051597:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574546.051599:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574546.051601:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574546.051602:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574546.051605:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574546.051607:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574546.051610:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574546.051613:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574546.051616:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574546.051619:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516711424, portal 4 00000100:00000001:0.0:1788574546.051622:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574546.051623:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788574546.051628:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516711424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051634:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574546.051637:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574546.051639:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574546.051641:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516711424, offset 0 00000400:00000200:0.0:1788574546.051645:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574546.051649:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574546.051653:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574546.051656:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574546.051658:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574546.051661:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574546.051663:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574546.051666:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547210:547210:256:4294967295] 192.168.202.128@tcp LPNI seq info [547133:547133:8:4294967295] 00000400:00000200:0.0:1788574546.051670:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574546.051676:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574546.051686:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574546.051689:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574546.051693:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574546.051695:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574546.051710:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574546.051712:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574546.051715:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574546.051719:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574546.051723:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574546.051724:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574546.051727:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574546.051731:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516711424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051739:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574546.051742:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516711424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574571 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051749:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.051751:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574546.051754:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574546.051757:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574546.051760:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574546.051763:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309906089 00000100:00000001:0.0:1788574546.051765:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309909742 00000100:00000001:0.0:1788574546.051767:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309906089 00000100:00000001:0.0:1788574546.051769:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309909742 00000100:00000001:0.0:1788574546.051772:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574546.051773:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.051775:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574546.051779:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574546/real 1788574546] req@ffff8fc4ac822a00 x1875440516711424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574546.051822:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574546.051824:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574546.051826:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574546.051829:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574546.051831:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574546.051833:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574546.051838:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516711424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574546.051846:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516711424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051853:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574546.051855:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.051857:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574546.051858:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574546.051862:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516711424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051869:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574546.051873:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516711424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051881:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574546.051883:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.051885:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574546.051886:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.051888:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574546.051890:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574546.051893:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574546.051897:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574546.051899:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574546.051908:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516711424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574546.051916:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516711424:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574546.051920:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.051922:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574546.051925:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516711040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051932:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574546.051934:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574546.051936:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574546.051939:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574546.051941:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574546.051942:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574546.051944:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574546.051946:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574546.051949:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574546.051950:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574546.051952:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788574546.051954:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574546.051956:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.051957:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574546.051960:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516711168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.051967:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574546.051968:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574546.051970:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574546.051972:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574546.051973:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574546.051975:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574546.051977:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574546.051978:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788574546.051980:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574546.051985:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574546.051987:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788574546.051990:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574546.051991:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.051993:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574546.051997:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516711296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.052004:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574546.052005:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574546.052007:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574546.052009:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574546.052011:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574546.052012:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574546.052014:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574546.052016:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788574546.052018:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574546.052019:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574546.052020:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788574546.052022:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574546.052023:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.052025:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574546.052029:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516711424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574571 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574546.052035:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574546.052037:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574546.052039:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574546.052041:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574546.052042:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574546.052043:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574546.052044:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574546.052046:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788574546.052049:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574546.052050:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574546.052051:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788574546.052054:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574546.052055:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.052060:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.052064:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574546.052065:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.052067:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574546.052069:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574546.560923:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574546.560933:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574546.560937:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574546.560946:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574546.560954:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574546.560966:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.561004:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574546.561010:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574546.561027:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574547.265069:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574547.265073:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574547.265076:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574547.584978:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574547.584995:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574547.585007:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574547.585018:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574547.585031:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574547.585042:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574547.585087:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574547.585098:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574547.585109:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574548.608971:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574548.608983:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574548.608988:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574548.608997:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574548.609005:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574548.609011:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574548.609034:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574548.609039:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574548.609044:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574549.313086:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574549.313092:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574549.313096:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574549.632949:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574549.632958:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574549.632960:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574549.632967:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574549.632975:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574549.632982:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574549.632987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574549.632992:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574549.633002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574549.888914:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309909742, at 4309909760 00000100:00000010:0.0:1788574549.888934:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574550.337092:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574550.337095:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574550.337098:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574550.656970:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574550.656977:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574550.656981:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574550.656991:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574550.657006:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574550.657014:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574550.657021:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574550.657024:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574550.657036:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574551.168965:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574551.168976:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574551.168980:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574551.168984:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574551.168991:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574551.168999:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15238 00000400:00000200:0.0:1788574551.169010:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574551.169018:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574551.169021:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574551.169025:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169029:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574551.169049:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169052:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574551.169057:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574551.169059:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574551.169063:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574551.169066:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574551.169069:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574551.169072:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574551.169075:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169078:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574551.169080:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169082:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574551.169085:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574551.169089:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169092:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574551.169095:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574551.169099:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574551.169102:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.169106:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574551.169109:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574551.169116:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574551.169120:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574551.169124:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574551.169126:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.169130:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574551.169133:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574551.169137:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574551.169145:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574551.169152:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574551.169163:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516711552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574551.169174:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574551.169184:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574551.169191:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516711552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574551.169219:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574551.169223:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574551.169227:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574551.169229:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574551.169231:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574551.169235:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574551.169239:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15238 00000400:00000200:0.0:1788574551.169245:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574551.169249:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574551.169252:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574551.169258:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169264:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574551.169275:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169281:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574551.169286:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574551.169291:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574551.169298:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574551.169302:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574551.169309:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574551.169315:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574551.169326:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169331:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574551.169341:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169345:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574551.169355:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574551.169368:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169387:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574551.169393:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574551.169402:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574551.169407:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.169413:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574551.169418:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574551.169423:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.169425:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574551.169430:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574551.169433:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574551.169439:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574551.169441:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.169444:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574551.169449:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574551.169458:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574551.169468:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574551.169477:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574551.169494:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516711680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574551.169517:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574551.169522:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574551.169541:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516711680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574551.169593:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574551.169600:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574551.169618:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574551.169629:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574551.169635:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574551.169640:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574551.169652:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15238 00000400:00000200:0.0:1788574551.169669:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574551.169695:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574551.169704:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574551.169715:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169722:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574551.169731:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169744:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574551.169757:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574551.169761:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574551.169765:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574551.169772:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574551.169782:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574551.169793:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574551.169876:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169888:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574551.169897:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169907:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574551.169918:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574551.169933:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574551.169946:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574551.169953:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574551.169963:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574551.169975:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.169983:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574551.169989:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574551.169999:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.170010:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574551.170021:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574551.170029:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574551.170043:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574551.170051:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.170062:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574551.170075:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574551.170082:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574551.170094:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574551.170107:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574551.170129:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516711808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574551.170160:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574551.170170:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574551.170184:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516711808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574551.170203:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574551.170213:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574551.170229:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574551.170233:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574551.170238:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574551.170245:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574551.170251:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15238 00000400:00000200:0.0:1788574551.170257:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574551.170269:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574551.170277:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574551.170285:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.170291:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574551.170299:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.170308:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574551.170318:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574551.170322:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574551.170327:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574551.170333:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574551.170341:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574551.170350:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574551.170371:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.170379:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574551.170386:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.170397:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574551.170405:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574551.170416:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574551.170425:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574551.170433:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574551.170441:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574551.170448:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.170456:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574551.170463:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574551.170474:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.170488:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574551.170500:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574551.170509:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574551.170521:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574551.170527:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.170537:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574551.170544:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574551.170550:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574551.170561:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574551.170575:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574551.170590:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516711936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574551.170627:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574551.170637:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574551.170654:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516711936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574551.170692:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574551.170701:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574551.170726:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574551.170734:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574551.170740:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574551.170752:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574551.170763:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15238 00000400:00000200:0.0:1788574551.170780:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574551.170795:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574551.170806:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574551.170856:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.170864:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574551.170875:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.170884:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574551.170900:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574551.170905:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574551.170914:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574551.170919:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574551.170928:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574551.170939:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574551.170951:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.170964:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574551.170974:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574551.170986:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574551.170997:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574551.171010:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574551.171018:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574551.171030:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574551.171038:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574551.171046:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.171057:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574551.171064:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574551.171085:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.171093:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574551.171099:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574551.171109:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574551.171120:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574551.171127:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574551.171136:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574551.171142:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574551.171147:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574551.171161:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574551.171173:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574551.171190:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516712064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574551.171244:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574551.171257:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574551.171275:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516712064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574551.171317:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574551.171330:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15248) 00000100:00000001:0.0:1788574551.681009:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574551.681014:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574551.681018:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574551.681024:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574551.681037:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574551.681048:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574551.681057:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574551.681061:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574551.681065:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574552.385090:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574552.385096:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574552.385101:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574552.704966:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574552.704973:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574552.704977:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574552.705000:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574552.705016:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574552.705021:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574552.705025:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574552.705028:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574552.705033:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574553.728936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574553.728942:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574553.728945:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574553.728950:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574553.728957:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574553.728962:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574553.728970:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574553.728974:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574553.728979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574554.432969:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574554.432973:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574554.432984:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574554.753012:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574554.753020:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574554.753023:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574554.753032:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574554.753041:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574554.753045:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574554.753048:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574554.753050:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574554.753053:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574555.008957:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309914862, at 4309914880 00000100:00000010:0.0:1788574555.008972:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abcc0. 00000020:00000001:0.0:1788574555.457068:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574555.457072:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574555.457075:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574555.776992:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574555.776997:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574555.777000:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574555.777006:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574555.777011:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574555.777028:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574555.777035:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574555.777039:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574555.777048:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574556.800904:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574556.800909:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574556.800912:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574556.800919:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574556.800924:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574556.800928:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574556.800931:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574556.800933:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574556.800939:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574557.504980:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574557.504985:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574557.504989:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574557.824957:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574557.824963:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574557.824968:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574557.824973:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574557.824979:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574557.824983:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574557.824993:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574557.824995:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574557.824997:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574558.848992:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574558.848998:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574558.849002:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574558.849008:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574558.849013:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574558.849019:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574558.849034:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574558.849038:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574558.849042:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574559.552995:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574559.552998:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574559.553001:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574559.872925:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574559.872945:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574559.872950:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574559.872959:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574559.872966:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574559.872971:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574559.872977:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574559.872981:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574559.872985:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574560.128964:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309919981, at 4309920000 00000100:00000010:0.0:1788574560.128982:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abc00. 00000020:00000001:0.0:1788574560.577012:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574560.577016:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574560.577018:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574560.896924:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574560.896929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574560.896934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574560.896939:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574560.896944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574560.896950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574560.896962:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574560.896966:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574560.896969:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574560.896984:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574560.896987:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574560.896989:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574560.896991:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574560.896995:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574560.896999:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574560.897004:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574560.897006:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574560.897010:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.409247:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574561.409253:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574561.409256:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574561.409266:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516712832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.409286:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574561.409300:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574561.409305:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574561.409311:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574561.409315:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574561.409320:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516712832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574561.409325:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574561.409330:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574561.409332:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574561.409337:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574561.409343:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574561.409348:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574561.409358:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574561.409364:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574561.409371:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574561.409380:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516712832, portal 25 00000100:00000001:0.0:1788574561.409383:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574561.409387:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788574561.409402:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516712832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.409424:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574561.409433:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574561.409440:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574561.409448:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516712832, offset 0 00000400:00000200:0.0:1788574561.409463:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574561.409476:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574561.409487:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574561.409498:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574561.409504:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574561.409510:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574561.409518:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574561.409529:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547221:547221:256:4294967295] 192.168.202.128@tcp LPNI seq info [547144:547144:8:4294967295] 00000400:00000200:0.0:1788574561.409542:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574561.409563:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574561.409588:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574561.409596:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574561.409610:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574561.409616:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574561.409651:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574561.409657:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574561.409668:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574561.409680:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574561.409689:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574561.409696:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574561.409706:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574561.409716:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516712832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.409745:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574561.409754:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516712832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.409772:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.409779:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574561.409787:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574561.409797:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574561.409804:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309925101 00000100:00000010:0.0:1788574561.409883:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574561.409894:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574561.409905:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309925102 00000100:00000010:0.0:1788574561.409912:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50960. 00000100:00000001:0.0:1788574561.409919:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574561.409927:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574561.409935:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.409942:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574561.409966:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574561/real 1788574561] req@ffff8fc4ac822d80 x1875440516712832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574561.410000:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574561.410002:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574561.410005:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574561.410008:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574561.410011:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574561.410014:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574561.410020:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516712832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574561.410028:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516712832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410035:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574561.410037:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.410039:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574561.410041:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574561.410045:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516712832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410052:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574561.410057:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516712832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410065:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574561.410067:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.410069:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574561.410070:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.410073:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574561.410075:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574561.410078:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574561.410082:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574561.410085:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574561.410089:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516712832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574561.410104:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516712832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574561.410110:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.410112:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574561.410124:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516712832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410132:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574561.410135:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574561.410138:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574561.410142:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574561.410144:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574561.410145:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574561.410148:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574561.410150:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788574561.410155:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574561.410157:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574561.410158:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788574561.410162:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574561.410163:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.410166:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.410173:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574561.410175:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.410177:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574561.410178:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574561.410180:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574561.410184:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516712960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410191:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574561.410193:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574561.410195:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574561.410196:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574561.410198:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574561.410201:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516712960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574561.410210:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574561.410212:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574561.410214:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574561.410215:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574561.410218:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574561.410220:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574561.410224:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574561.410226:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574561.410230:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574561.410234:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516712960, portal 10 00000100:00000001:0.0:1788574561.410236:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574561.410237:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788574561.410244:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516712960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410251:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574561.410253:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574561.410256:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574561.410258:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516712960, offset 0 00000400:00000200:0.0:1788574561.410262:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574561.410266:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574561.410270:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574561.410273:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574561.410276:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574561.410279:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574561.410281:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574561.410284:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547222:547222:256:4294967295] 192.168.202.128@tcp LPNI seq info [547145:547145:8:4294967295] 00000400:00000200:0.0:1788574561.410289:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574561.410295:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574561.410299:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574561.410303:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574561.410306:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574561.410315:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574561.410326:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574561.410328:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574561.410331:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574561.410335:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574561.410339:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574561.410341:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574561.410344:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574561.410347:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516712960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410355:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574561.410358:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516712960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410364:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.410366:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574561.410370:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574561.410374:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574561.410378:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574561.410380:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309921299 00000100:00000001:0.0:1788574561.410383:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309925101 00000100:00000001:0.0:1788574561.410385:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309921299 00000100:00000001:0.0:1788574561.410387:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309925102 00000100:00000001:0.0:1788574561.410389:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574561.410392:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.410394:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574561.410397:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574561/real 1788574561] req@ffff8fc4ac821f80 x1875440516712960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574561.410405:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574561.410406:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574561.410413:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574561.410415:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574561.410417:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574561.410420:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574561.410423:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516712960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574561.410432:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516712960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410439:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574561.410440:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.410442:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574561.410444:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574561.410447:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516712960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410455:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574561.410459:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516712960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410466:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574561.410468:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.410470:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574561.410471:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.410473:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574561.410475:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574561.410478:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574561.410481:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574561.410483:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574561.410488:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516712960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574561.410496:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516712960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574561.410506:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574561.410509:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516713088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410516:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574561.410517:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574561.410520:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574561.410522:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574561.410524:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574561.410526:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516713088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574561.410530:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574561.410532:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574561.410533:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574561.410535:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574561.410538:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574561.410540:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574561.410544:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574561.410547:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574561.410551:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574561.410554:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516713088, portal 10 00000100:00000001:0.0:1788574561.410556:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574561.410558:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788574561.410564:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516713088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410571:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574561.410574:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574561.410576:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574561.410578:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516713088, offset 0 00000400:00000200:0.0:1788574561.410583:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574561.410587:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574561.410590:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574561.410593:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574561.410601:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574561.410604:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574561.410606:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574561.410609:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547223:547223:256:4294967295] 192.168.202.128@tcp LPNI seq info [547146:547146:8:4294967295] 00000400:00000200:0.0:1788574561.410613:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574561.410619:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574561.410624:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574561.410629:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574561.410633:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574561.410635:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574561.410646:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574561.410648:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574561.410651:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574561.410654:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574561.410657:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574561.410659:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574561.410662:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574561.410666:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516713088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410674:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574561.410678:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516713088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410685:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.410686:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574561.410689:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574561.410692:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574561.410696:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574561.410698:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309921306 00000100:00000001:0.0:1788574561.410700:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309925101 00000100:00000001:0.0:1788574561.410703:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309921306 00000100:00000001:0.0:1788574561.410710:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309925102 00000100:00000001:0.0:1788574561.410712:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574561.410714:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.410716:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574561.410720:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574561/real 1788574561] req@ffff8fc4ac823480 x1875440516713088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574561.410728:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574561.410730:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574561.410731:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574561.410733:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574561.410735:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574561.410737:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574561.410740:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516713088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574561.410749:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516713088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410755:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574561.410756:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.410758:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574561.410759:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574561.410763:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516713088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410770:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574561.410774:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516713088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410780:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574561.410783:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.410785:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574561.410786:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.410791:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574561.410794:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574561.410797:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574561.410800:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574561.410802:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574561.410806:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516713088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574561.410849:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516713088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574561.410854:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574561.410857:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516713216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410865:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574561.410866:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574561.410868:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574561.410871:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574561.410872:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574561.410875:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516713216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574561.410879:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574561.410880:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574561.410882:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574561.410884:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574561.410887:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574561.410889:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574561.410893:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574561.410896:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574561.410899:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574561.410902:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516713216, portal 4 00000100:00000001:0.0:1788574561.410904:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574561.410905:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788574561.410911:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516713216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.410922:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574561.410925:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574561.410927:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574561.410929:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516713216, offset 0 00000400:00000200:0.0:1788574561.410933:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574561.410938:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574561.410941:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574561.410944:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574561.410947:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574561.410949:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574561.410951:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574561.410954:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547224:547224:256:4294967295] 192.168.202.128@tcp LPNI seq info [547147:547147:8:4294967295] 00000400:00000200:0.0:1788574561.410959:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574561.410966:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574561.410971:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574561.410974:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574561.410978:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574561.410981:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574561.410992:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574561.410994:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574561.410997:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574561.411001:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574561.411004:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574561.411006:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574561.411009:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574561.411012:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516713216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411021:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574561.411025:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516713216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411042:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.411044:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574561.411047:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574561.411050:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574561.411054:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574561.411056:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309921447 00000100:00000001:0.0:1788574561.411058:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309925101 00000100:00000001:0.0:1788574561.411060:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309921447 00000100:00000001:0.0:1788574561.411062:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309925102 00000100:00000001:0.0:1788574561.411064:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574561.411066:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.411068:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574561.411072:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574561/real 1788574561] req@ffff8fc4ac822300 x1875440516713216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574561.411079:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574561.411080:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574561.411082:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574561.411085:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574561.411086:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574561.411088:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574561.411091:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516713216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574561.411099:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516713216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411106:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574561.411107:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.411109:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574561.411110:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574561.411114:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516713216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411125:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574561.411129:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516713216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411135:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574561.411138:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.411139:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574561.411140:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.411142:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574561.411144:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574561.411147:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574561.411150:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574561.411152:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574561.411156:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516713216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574561.411164:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516713216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574561.411167:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574561.411171:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516713344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411177:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574561.411179:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574561.411181:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574561.411183:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574561.411184:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574561.411187:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516713344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574561.411190:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574561.411192:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574561.411198:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574561.411199:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574561.411202:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574561.411205:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574561.411208:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574561.411211:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574561.411214:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574561.411217:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516713344, portal 4 00000100:00000001:0.0:1788574561.411219:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574561.411220:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788574561.411225:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516713344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411232:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574561.411234:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574561.411236:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574561.411238:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516713344, offset 0 00000400:00000200:0.0:1788574561.411241:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574561.411245:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574561.411248:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574561.411251:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574561.411253:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574561.411255:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574561.411258:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574561.411262:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547225:547225:256:4294967295] 192.168.202.128@tcp LPNI seq info [547148:547148:8:4294967295] 00000400:00000200:0.0:1788574561.411266:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574561.411272:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574561.411276:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574561.411279:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574561.411283:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574561.411286:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574561.411296:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574561.411302:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574561.411306:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574561.411310:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574561.411313:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574561.411314:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574561.411316:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574561.411320:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516713344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411327:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574561.411329:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516713344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574586 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411336:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.411337:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574561.411340:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574561.411343:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574561.411346:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574561.411348:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309921449 00000100:00000001:0.0:1788574561.411352:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309925101 00000100:00000001:0.0:1788574561.411354:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309921449 00000100:00000001:0.0:1788574561.411356:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309925102 00000100:00000001:0.0:1788574561.411358:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574561.411360:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.411362:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574561.411365:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574561/real 1788574561] req@ffff8fc4ac820700 x1875440516713344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574561.411372:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574561.411374:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574561.411376:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574561.411378:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574561.411379:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574561.411386:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574561.411389:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516713344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574561.411397:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516713344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411404:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574561.411406:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.411409:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574561.411410:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574561.411413:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516713344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411420:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574561.411423:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516713344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411429:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574561.411431:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.411433:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574561.411434:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.411436:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574561.411438:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574561.411440:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574561.411443:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574561.411445:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574561.411450:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516713344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574561.411456:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516713344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574561.411459:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.411461:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574561.411468:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516712960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411477:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574561.411478:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574561.411480:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574561.411483:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574561.411484:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574561.411486:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574561.411488:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574561.411490:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788574561.411493:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574561.411494:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574561.411495:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788574561.411498:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574561.411500:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.411502:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574561.411506:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516713088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411512:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574561.411513:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574561.411515:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574561.411518:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574561.411519:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574561.411520:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574561.411523:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574561.411524:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788574561.411526:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574561.411527:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574561.411529:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788574561.411531:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574561.411532:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.411533:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574561.411536:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516713216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411542:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574561.411548:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574561.411549:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574561.411552:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574561.411554:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574561.411555:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574561.411557:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574561.411559:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788574561.411561:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574561.411562:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574561.411564:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788574561.411566:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574561.411568:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.411570:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574561.411574:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516713344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574586 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574561.411580:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574561.411581:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574561.411583:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574561.411585:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574561.411586:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574561.411587:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574561.411589:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574561.411590:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574561.411592:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574561.411593:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574561.411594:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788574561.411596:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574561.411597:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.411599:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.411602:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574561.411603:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.411605:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574561.411606:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574561.920959:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574561.920975:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574561.920989:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574561.921005:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574561.921034:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574561.921043:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.921089:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574561.921092:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574561.921094:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574562.624983:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574562.624986:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574562.624989:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574563.968993:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574563.968999:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574563.969010:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574563.969026:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574563.969042:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574563.969047:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574563.969059:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574563.969066:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574563.969074:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574564.673010:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574564.673013:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574564.673015:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574564.992914:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574564.992927:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574564.992931:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574564.992937:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574564.992943:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574564.992947:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574564.992968:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574564.992972:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574564.992976:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574565.248893:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309925101, at 4309925120 00000100:00000010:0.0:1788574565.248902:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574565.696986:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574565.696989:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574565.696992:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574566.016930:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574566.016939:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574566.016941:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574566.016961:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574566.016970:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574566.016976:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574566.016986:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574566.016990:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574566.016995:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574566.528930:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574566.528940:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574566.528944:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574566.528948:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574566.528955:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574566.528962:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15253 00000400:00000200:0.0:1788574566.528973:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574566.528983:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574566.528987:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574566.528991:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.528996:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574566.529003:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529007:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574566.529011:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574566.529015:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574566.529018:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574566.529022:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574566.529025:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574566.529028:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574566.529033:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529036:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574566.529039:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529042:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574566.529046:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574566.529053:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529075:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574566.529078:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574566.529081:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574566.529083:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.529084:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574566.529086:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574566.529091:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574566.529094:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574566.529097:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574566.529099:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.529101:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574566.529103:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574566.529108:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574566.529115:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574566.529121:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574566.529129:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516713472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574566.529136:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574566.529140:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574566.529143:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516713472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574566.529162:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574566.529164:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574566.529167:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574566.529168:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574566.529169:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574566.529171:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574566.529173:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15253 00000400:00000200:0.0:1788574566.529176:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574566.529180:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574566.529181:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574566.529183:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529191:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574566.529195:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529198:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574566.529202:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574566.529203:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574566.529205:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574566.529208:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574566.529212:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574566.529216:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574566.529221:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529224:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574566.529227:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529229:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574566.529232:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574566.529235:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529236:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574566.529237:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574566.529242:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574566.529247:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.529250:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574566.529255:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574566.529260:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.529262:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574566.529267:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574566.529272:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574566.529276:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574566.529281:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.529285:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574566.529289:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574566.529293:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574566.529300:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574566.529307:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574566.529325:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516713600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574566.529340:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574566.529343:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574566.529351:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516713600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574566.529367:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574566.529372:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574566.529376:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574566.529379:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574566.529382:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574566.529385:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574566.529393:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15253 00000400:00000200:0.0:1788574566.529403:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574566.529410:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574566.529413:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574566.529416:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529421:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574566.529426:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529431:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574566.529435:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574566.529437:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574566.529439:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574566.529440:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574566.529444:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574566.529446:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574566.529450:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529453:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574566.529457:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529467:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574566.529472:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574566.529475:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529477:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574566.529478:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574566.529481:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574566.529482:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.529484:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574566.529486:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574566.529491:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.529493:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574566.529497:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574566.529501:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574566.529506:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574566.529507:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.529512:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574566.529516:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574566.529518:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574566.529526:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574566.529530:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574566.529540:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516713728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574566.529553:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574566.529557:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574566.529563:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516713728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574566.529577:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574566.529582:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574566.529586:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574566.529588:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574566.529591:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574566.529601:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574566.529608:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15253 00000400:00000200:0.0:1788574566.529616:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574566.529624:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574566.529628:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574566.529633:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529636:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574566.529641:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529644:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574566.529647:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574566.529649:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574566.529653:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574566.529657:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574566.529661:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574566.529664:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574566.529669:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529673:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574566.529677:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529681:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574566.529684:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574566.529690:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529694:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574566.529699:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574566.529703:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574566.529709:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.529712:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574566.529717:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574566.529722:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.529726:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574566.529732:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574566.529744:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574566.529748:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574566.529751:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.529753:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574566.529757:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574566.529760:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574566.529767:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574566.529772:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574566.529801:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516713856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574566.529853:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574566.529859:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574566.529868:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516713856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574566.529888:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574566.529894:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574566.529898:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574566.529900:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574566.529901:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574566.529904:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574566.529908:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15253 00000400:00000200:0.0:1788574566.529916:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574566.529928:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574566.529933:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574566.529941:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529944:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574566.529950:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.529954:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574566.529960:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574566.529963:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574566.529974:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574566.529979:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574566.529984:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574566.529990:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574566.529999:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.530004:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574566.530009:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574566.530016:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574566.530022:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574566.530028:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574566.530033:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574566.530037:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574566.530042:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574566.530046:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.530050:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574566.530055:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574566.530059:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.530064:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574566.530071:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574566.530076:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574566.530080:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574566.530085:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574566.530089:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574566.530091:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574566.530093:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574566.530099:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574566.530106:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574566.530115:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516713984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574566.530131:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574566.530134:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574566.530143:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516713984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574566.530165:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574566.530174:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15264) 00000100:00000001:0.0:1788574567.040896:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574567.040905:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574567.040909:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574567.040918:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574567.040925:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574567.040929:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574567.040946:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574567.040949:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574567.040951:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574567.040982:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574567.040987:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574567.040989:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574567.040992:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574567.040996:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574567.040999:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574567.041002:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574567.041004:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574567.041006:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574567.745003:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574567.745007:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574567.745010:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574568.064938:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574568.064944:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574568.064948:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574568.064955:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574568.064960:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574568.064966:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574568.064970:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574568.064974:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574568.064978:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574569.088899:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574569.088910:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574569.088914:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574569.088923:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574569.088945:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574569.088948:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.088973:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574569.088977:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.088981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574569.601568:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574569.601573:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574569.601577:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574569.601585:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574566/real 1788574569] req@ffff8fc4b80c0380 x1875440516713856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574569.601599:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574569.601601:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574569.601604:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a0cb8. 00000400:00000200:0.0:1788574569.601608:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed7f8 00000400:00000010:0.0:1788574569.601610:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed7f8. 00000100:00000001:0.0:1788574569.601613:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574569.601617:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c0380 x1875440516713856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574569.601625:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c0380 x1875440516713856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.601631:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574569.601633:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.601636:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574569.601639:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574569.601643:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c0380 x1875440516713856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.601650:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574569.601655:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c0380 x1875440516713856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.601662:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574569.601665:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.601673:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574569.601674:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.601677:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574569.601680:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574569.601684:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574569.601689:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574569.601693:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574569.601697:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c0380 x1875440516713856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574569.601706:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516713856:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574569.601712:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574569.601715:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574566/real 1788574569] req@ffff8fc4b80c0a80 x1875440516713984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574569.601724:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574569.601725:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574569.601727:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a0c08. 00000400:00000200:0.0:1788574569.601730:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed4c8 00000400:00000010:0.0:1788574569.601732:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed4c8. 00000100:00000001:0.0:1788574569.601735:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574569.601738:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c0a80 x1875440516713984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574569.601747:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c0a80 x1875440516713984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.601755:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574569.601756:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.601758:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574569.601775:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574569.601780:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c0a80 x1875440516713984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.601801:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574569.601806:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c0a80 x1875440516713984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.601907:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574569.601910:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.601913:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574569.601914:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.601917:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574569.601919:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574569.601922:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574569.601928:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574569.601931:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574569.601936:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c0a80 x1875440516713984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574569.601944:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516713984:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574569.601949:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574569.601953:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574566/real 1788574569] req@ffff8fc4b80c1500 x1875440516713600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574569.601962:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574569.601963:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574569.601965:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a0840. 00000400:00000200:0.0:1788574569.601968:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76eddd0 00000400:00000010:0.0:1788574569.601970:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76eddd0. 00000100:00000001:0.0:1788574569.601974:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574569.601977:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c1500 x1875440516713600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574569.601999:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c1500 x1875440516713600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602006:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574569.602008:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602010:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574569.602012:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574569.602016:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c1500 x1875440516713600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602023:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574569.602027:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c1500 x1875440516713600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602034:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574569.602037:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602039:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574569.602040:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602042:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574569.602044:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574569.602047:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574569.602051:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574569.602054:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574569.602058:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c1500 x1875440516713600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574569.602066:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516713600:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574569.602071:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574569.602075:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574566/real 1788574569] req@ffff8fc4b80c1180 x1875440516713728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574569.602097:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574569.602099:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574569.602115:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a09a0. 00000400:00000200:0.0:1788574569.602119:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed440 00000400:00000010:0.0:1788574569.602120:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed440. 00000100:00000001:0.0:1788574569.602123:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574569.602127:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c1180 x1875440516713728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574569.602137:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c1180 x1875440516713728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602143:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574569.602145:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602146:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574569.602148:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574569.602152:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c1180 x1875440516713728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602159:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574569.602163:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c1180 x1875440516713728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602169:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574569.602172:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602174:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574569.602175:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602177:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574569.602179:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574569.602183:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574569.602186:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574569.602189:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574569.602199:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c1180 x1875440516713728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574569.602206:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516713728:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574569.602211:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574569.602214:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574566/real 1788574569] req@ffff8fc4b80c3b80 x1875440516713472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574569.602223:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574569.602225:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574569.602227:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a0948. 00000400:00000200:0.0:1788574569.602230:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed220 00000400:00000010:0.0:1788574569.602232:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed220. 00000100:00000001:0.0:1788574569.602235:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574569.602238:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c3b80 x1875440516713472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574569.602246:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c3b80 x1875440516713472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602253:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574569.602255:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602257:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574569.602258:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574569.602262:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c3b80 x1875440516713472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602268:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574569.602271:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c3b80 x1875440516713472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602278:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574569.602281:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602283:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574569.602290:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602292:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574569.602293:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574569.602296:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574569.602301:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574569.602303:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574569.602308:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c3b80 x1875440516713472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574569.602318:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516713472:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574569.602324:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602326:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574569.602330:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c0380 x1875440516713856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602337:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574569.602340:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574569.602344:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788574569.602349:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574569.602352:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574569.602353:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574569.602356:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574569.602359:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574569.602364:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574569.602366:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574569.602368:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c0380. 00000100:00000001:0.0:1788574569.602373:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574569.602374:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602376:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574569.602380:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c0a80 x1875440516713984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602386:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574569.602388:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574569.602390:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788574569.602399:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574569.602400:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574569.602401:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574569.602404:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574569.602406:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574569.602408:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574569.602410:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574569.602411:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c0a80. 00000100:00000001:0.0:1788574569.602414:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574569.602415:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602417:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574569.602421:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c1500 x1875440516713600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602428:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574569.602429:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574569.602431:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788574569.602433:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574569.602434:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574569.602434:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574569.602436:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574569.602437:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574569.602438:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574569.602439:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574569.602440:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c1500. 00000100:00000001:0.0:1788574569.602442:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574569.602443:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602444:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574569.602447:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c1180 x1875440516713728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602451:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574569.602451:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574569.602452:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788574569.602453:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574569.602454:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574569.602455:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574569.602456:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574569.602461:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574569.602463:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574569.602464:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574569.602465:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c1180. 00000100:00000001:0.0:1788574569.602468:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574569.602469:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602471:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574569.602474:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c3b80 x1875440516713472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574591 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574569.602481:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574569.602482:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574569.602484:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788574569.602487:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574569.602489:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574569.602490:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574569.602492:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574569.602493:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574569.602495:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574569.602496:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574569.602498:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c3b80. 00000100:00000001:0.0:1788574569.602499:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574569.602501:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602503:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574569.602512:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574569.602514:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574569.602516:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574569.602517:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574569.793025:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574569.793029:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574569.793033:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574570.112913:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574570.112919:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574570.112923:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574570.112934:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574570.112941:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574570.112948:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574570.112961:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574570.112983:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574570.112989:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574570.368876:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309930221, at 4309930240 00000100:00000010:0.0:1788574570.368895:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab3c0. 00000020:00000001:0.0:1788574570.817006:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574570.817012:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574570.817016:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574571.136925:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574571.136938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574571.136941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574571.136950:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574571.136956:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574571.136963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.136971:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574571.136976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.136980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.137060:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574571.137065:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574571.137067:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574571.137071:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574571.137076:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574571.137080:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.137085:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574571.137087:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.137091:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574571.649211:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788574571.649315:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574571.649328:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574571.649338:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574571.649362:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516714112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.649409:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574571.649415:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574571.649425:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574571.649436:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574571.649446:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574571.649483:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516714112:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574571.649496:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574571.649510:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574571.649518:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574571.649525:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574571.649534:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574571.649539:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574571.649560:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574571.649569:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574571.649590:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574571.649610:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516714112, portal 25 00000100:00000001:0.0:1788574571.649621:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574571.649628:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788574571.649654:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516714112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.649700:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574571.649714:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574571.649721:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574571.649734:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516714112, offset 0 00000400:00000200:0.0:1788574571.649767:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574571.649786:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574571.649802:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574571.649854:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574571.649864:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574571.649881:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574571.649893:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574571.649905:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547231:547231:256:4294967295] 192.168.202.128@tcp LPNI seq info [547154:547154:8:4294967295] 00000400:00000200:0.0:1788574571.649918:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574571.649949:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574571.649975:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574571.649989:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574571.650024:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574571.650038:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574571.650080:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574571.650086:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574571.650097:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574571.650112:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574571.650123:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574571.650132:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574571.650147:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574571.650159:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516714112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.650195:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574571.650212:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516714112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.650226:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.650228:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574571.650247:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574571.650263:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574571.650272:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309935342 00000100:00000010:0.0:1788574571.650285:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574571.650302:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574571.650316:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309935342 00000100:00000010:0.0:1788574571.650328:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50900. 00000100:00000001:0.0:1788574571.650340:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574571.650351:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574571.650361:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.650368:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574571.650381:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574571/real 1788574571] req@ffff8fc4ac821c00 x1875440516714112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574571.650406:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574571.650424:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574571.650431:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574571.650446:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574571.650452:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574571.650459:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574571.650470:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516714112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574571.650497:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516714112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.650515:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574571.650521:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.650527:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574571.650531:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574571.650544:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516714112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.650583:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574571.650602:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516714112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.650631:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574571.650640:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.650645:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574571.650650:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.650657:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574571.650663:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574571.650672:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574571.650686:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574571.650696:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574571.650710:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516714112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574571.650725:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516714112:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574571.650763:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.650773:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574571.650784:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516714112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.650857:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574571.650868:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574571.650875:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574571.650885:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574571.650891:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574571.650897:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574571.650904:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574571.650910:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862ac00. 02000000:00000001:0.0:1788574571.650922:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574571.650929:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574571.650935:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788574571.650943:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574571.650948:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.650964:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.650981:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574571.650987:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.650994:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574571.650999:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574571.651003:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574571.651016:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516714240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651046:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574571.651052:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574571.651059:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574571.651063:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574571.651074:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574571.651090:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516714240:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574571.651098:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574571.651104:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574571.651106:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574571.651117:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574571.651121:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574571.651125:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574571.651130:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574571.651133:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574571.651138:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574571.651141:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516714240, portal 10 00000100:00000001:0.0:1788574571.651143:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574571.651145:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788574571.651153:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516714240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651159:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574571.651161:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574571.651162:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574571.651165:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516714240, offset 0 00000400:00000200:0.0:1788574571.651169:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574571.651173:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574571.651177:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574571.651180:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574571.651183:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574571.651185:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574571.651187:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574571.651191:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547232:547232:256:4294967295] 192.168.202.128@tcp LPNI seq info [547155:547155:8:4294967295] 00000400:00000200:0.0:1788574571.651193:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574571.651201:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574571.651207:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574571.651211:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574571.651218:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574571.651221:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574571.651236:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574571.651238:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574571.651248:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574571.651253:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574571.651257:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574571.651259:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574571.651262:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574571.651266:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516714240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651273:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574571.651277:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516714240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651284:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.651286:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574571.651290:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574571.651295:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574571.651299:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574571.651301:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309931540 00000100:00000001:0.0:1788574571.651305:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309935342 00000100:00000001:0.0:1788574571.651307:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309931540 00000100:00000001:0.0:1788574571.651310:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309935342 00000100:00000001:0.0:1788574571.651313:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574571.651315:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.651317:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574571.651322:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574571/real 1788574571] req@ffff8fc4ac820a80 x1875440516714240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574571.651330:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574571.651332:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574571.651333:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574571.651335:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574571.651336:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574571.651338:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574571.651347:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516714240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574571.651353:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516714240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651358:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574571.651359:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.651361:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574571.651361:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574571.651364:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516714240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651368:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574571.651370:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516714240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651375:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574571.651377:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.651379:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574571.651380:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.651381:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574571.651382:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574571.651384:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574571.651386:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574571.651388:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574571.651390:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516714240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574571.651395:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516714240:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574571.651399:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574571.651402:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516714368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651417:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574571.651419:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574571.651422:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574571.651424:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574571.651426:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574571.651430:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516714368:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574571.651433:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574571.651436:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574571.651438:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574571.651440:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574571.651444:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574571.651446:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574571.651450:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574571.651453:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574571.651457:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574571.651460:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516714368, portal 10 00000100:00000001:0.0:1788574571.651463:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574571.651464:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788574571.651471:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516714368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651478:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574571.651480:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574571.651482:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574571.651485:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516714368, offset 0 00000400:00000200:0.0:1788574571.651487:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574571.651491:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574571.651493:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574571.651496:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574571.651498:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574571.651499:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574571.651501:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574571.651508:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547233:547233:256:4294967295] 192.168.202.128@tcp LPNI seq info [547156:547156:8:4294967295] 00000400:00000200:0.0:1788574571.651513:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574571.651520:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574571.651525:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574571.651529:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574571.651533:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574571.651536:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574571.651556:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574571.651558:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574571.651561:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574571.651564:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574571.651567:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574571.651568:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574571.651570:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574571.651573:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516714368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651580:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574571.651582:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516714368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651587:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.651588:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574571.651590:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574571.651593:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574571.651595:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574571.651597:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309931547 00000100:00000001:0.0:1788574571.651599:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309935342 00000100:00000001:0.0:1788574571.651600:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309931547 00000100:00000001:0.0:1788574571.651602:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309935342 00000100:00000001:0.0:1788574571.651603:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574571.651609:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.651611:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574571.651613:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574571/real 1788574571] req@ffff8fc4ac823800 x1875440516714368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574571.651620:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574571.651621:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574571.651622:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574571.651624:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574571.651625:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574571.651627:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574571.651629:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516714368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574571.651634:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516714368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651638:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574571.651639:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.651640:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574571.651641:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574571.651644:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516714368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651648:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574571.651651:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516714368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651655:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574571.651657:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.651658:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574571.651658:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.651660:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574571.651662:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574571.651663:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574571.651669:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574571.651670:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574571.651673:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516714368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574571.651680:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516714368:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574571.651684:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574571.651688:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516714496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651696:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574571.651698:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574571.651701:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574571.651703:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574571.651706:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574571.651709:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516714496:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574571.651712:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574571.651715:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574571.651717:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574571.651719:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574571.651722:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574571.651725:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574571.651729:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574571.651732:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574571.651735:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574571.651738:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516714496, portal 4 00000100:00000001:0.0:1788574571.651739:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574571.651740:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788574571.651746:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516714496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651771:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574571.651781:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574571.651783:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574571.651787:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516714496, offset 0 00000400:00000200:0.0:1788574571.651791:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574571.651796:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574571.651799:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574571.651802:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574571.651805:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574571.651834:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574571.651837:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574571.651840:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547234:547234:256:4294967295] 192.168.202.128@tcp LPNI seq info [547157:547157:8:4294967295] 00000400:00000200:0.0:1788574571.651845:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574571.651852:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574571.651858:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574571.651862:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574571.651867:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574571.651869:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574571.651876:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574571.651878:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574571.651880:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574571.651883:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574571.651885:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574571.651886:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574571.651889:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574571.651891:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516714496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651901:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574571.651905:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516714496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651912:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.651914:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574571.651923:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574571.651927:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574571.651931:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574571.651933:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309931688 00000100:00000001:0.0:1788574571.651936:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309935342 00000100:00000001:0.0:1788574571.651939:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309931688 00000100:00000001:0.0:1788574571.651941:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309935342 00000100:00000001:0.0:1788574571.651944:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574571.651946:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.651948:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574571.651953:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574571/real 1788574571] req@ffff8fc4ac820e00 x1875440516714496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574571.651962:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574571.651964:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574571.651966:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574571.651968:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574571.651970:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574571.651972:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574571.651976:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516714496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574571.651985:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516714496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.651992:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574571.651994:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.651996:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574571.651998:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574571.652002:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516714496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652009:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574571.652015:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516714496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652030:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574571.652033:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652035:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574571.652037:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652039:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574571.652041:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574571.652044:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574571.652048:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574571.652051:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574571.652054:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516714496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574571.652060:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516714496:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574571.652063:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652064:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574571.652067:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516714240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652073:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574571.652075:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574571.652077:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574571.652086:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574571.652087:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574571.652089:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574571.652092:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574571.652094:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788574571.652097:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574571.652099:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574571.652100:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788574571.652104:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574571.652106:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652113:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574571.652117:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516714368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652124:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574571.652126:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574571.652128:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574571.652131:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574571.652132:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574571.652133:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574571.652135:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574571.652137:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788574571.652140:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574571.652141:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574571.652143:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788574571.652146:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574571.652147:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652148:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574571.652152:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516714496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652157:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574571.652158:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574571.652159:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574571.652161:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574571.652161:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574571.652162:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574571.652163:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574571.652164:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788574571.652166:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574571.652167:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574571.652168:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788574571.652169:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574571.652170:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652171:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652175:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574571.652176:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.652178:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574571.652183:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574571.652184:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574571.652188:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516714624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652193:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574571.652194:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574571.652196:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574571.652198:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574571.652201:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574571.652204:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516714624:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574571.652208:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574571.652210:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574571.652212:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574571.652214:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574571.652217:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574571.652220:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574571.652223:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574571.652226:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574571.652230:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574571.652234:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516714624, portal 4 00000100:00000001:0.0:1788574571.652236:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574571.652238:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788574571.652244:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516714624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652252:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574571.652255:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574571.652257:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574571.652258:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516714624, offset 0 00000400:00000200:0.0:1788574571.652261:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574571.652264:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574571.652267:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574571.652270:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574571.652272:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574571.652279:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574571.652282:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574571.652285:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547235:547235:256:4294967295] 192.168.202.128@tcp LPNI seq info [547158:547158:8:4294967295] 00000400:00000200:0.0:1788574571.652290:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574571.652296:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574571.652302:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574571.652305:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574571.652310:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574571.652312:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574571.652325:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574571.652327:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574571.652329:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574571.652331:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574571.652334:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574571.652335:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574571.652337:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574571.652340:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516714624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652346:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574571.652348:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516714624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574596 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652352:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.652353:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574571.652356:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574571.652358:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574571.652360:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574571.652362:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309931690 00000100:00000001:0.0:1788574571.652364:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309935342 00000100:00000001:0.0:1788574571.652366:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309931690 00000100:00000001:0.0:1788574571.652372:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309935342 00000100:00000001:0.0:1788574571.652374:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574571.652375:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.652377:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574571.652380:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574571/real 1788574571] req@ffff8fc4ac821880 x1875440516714624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574571.652386:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574571.652387:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574571.652389:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574571.652391:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574571.652392:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574571.652393:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574571.652395:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516714624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574571.652401:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516714624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652405:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574571.652406:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652408:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574571.652409:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574571.652411:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516714624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652415:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574571.652417:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516714624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652421:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574571.652423:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652424:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574571.652424:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652429:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574571.652431:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574571.652434:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574571.652438:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574571.652440:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574571.652445:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516714624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574571.652453:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516714624:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574571.652457:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652459:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574571.652462:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516714624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574596 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574571.652469:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574571.652471:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574571.652473:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574571.652476:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574571.652477:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574571.652479:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574571.652481:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574571.652483:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788574571.652486:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574571.652487:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574571.652489:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788574571.652492:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574571.652493:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652495:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574571.652499:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574571.652501:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574571.652503:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574571.652505:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574572.161007:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574572.161018:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574572.161037:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574572.161043:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574572.161048:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574572.161053:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574572.161060:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574572.161065:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574572.161074:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574572.865045:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574572.865049:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574572.865053:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574573.184972:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574573.184984:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574573.184990:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574573.185003:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574573.185015:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574573.185021:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574573.185055:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574573.185060:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574573.185066:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574574.208951:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574574.208955:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574574.208956:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574574.208960:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574574.208963:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574574.208966:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574574.208971:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574574.208975:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574574.208979:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574574.913096:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574574.913100:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574574.913104:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574575.232988:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574575.232994:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574575.232999:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574575.233008:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574575.233014:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574575.233019:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574575.233023:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574575.233033:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574575.233036:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574575.488883:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309935342, at 4309935360 00000100:00000010:0.0:1788574575.488895:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574575.936947:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574575.936950:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574575.936953:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574576.256911:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574576.256918:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574576.256922:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574576.256929:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574576.256933:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574576.256939:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.256945:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574576.256950:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.256953:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.769287:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574576.769298:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574576.769305:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574576.769323:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516714752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.769358:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574576.769364:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574576.769372:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574576.769378:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574576.769391:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574576.769400:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516714752:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574576.769414:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574576.769421:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574576.769424:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574576.769432:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574576.769446:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574576.769454:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574576.769474:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574576.769483:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574576.769504:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574576.769512:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516714752, portal 25 00000100:00000001:0.0:1788574576.769516:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574576.769520:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788574576.769530:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516714752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.769542:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574576.769552:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574576.769557:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574576.769567:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516714752, offset 0 00000400:00000200:0.0:1788574576.769582:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574576.769601:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574576.769622:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574576.769635:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574576.769651:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574576.769661:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574576.769670:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574576.769683:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547236:547236:256:4294967295] 192.168.202.128@tcp LPNI seq info [547159:547159:8:4294967295] 00000400:00000200:0.0:1788574576.769700:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574576.769751:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574576.769769:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574576.769776:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574576.769795:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574576.769799:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574576.769874:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574576.769883:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574576.769895:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574576.769910:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574576.769923:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574576.769928:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574576.769937:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574576.769959:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516714752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.769981:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574576.769989:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516714752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.770015:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.770021:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574576.770032:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574576.770041:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574576.770047:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309940462 00000100:00000010:0.0:1788574576.770057:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574576.770068:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574576.770079:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309940462 00000100:00000010:0.0:1788574576.770090:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50180. 00000100:00000001:0.0:1788574576.770098:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574576.770105:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574576.770114:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.770132:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574576.770145:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574576/real 1788574576] req@ffff8fc4901a5880 x1875440516714752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574576.770177:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574576.770180:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574576.770188:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574576.770201:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574576.770205:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574576.770214:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574576.770225:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516714752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574576.770242:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516714752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.770260:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574576.770271:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.770277:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574576.770281:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574576.770292:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516714752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.770306:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574576.770314:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516714752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.770337:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574576.770347:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.770353:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574576.770356:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.770359:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574576.770361:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574576.770373:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574576.770388:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574576.770394:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574576.770407:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516714752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574576.770436:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516714752:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574576.770449:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.770454:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574576.770464:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516714752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.770496:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574576.770500:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574576.770506:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574576.770516:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574576.770521:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574576.770534:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574576.770542:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574576.770550:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788574576.770561:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574576.770566:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574576.770572:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788574576.770582:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574576.770587:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.770592:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.770604:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574576.770609:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.770620:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574576.770624:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574576.770627:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574576.770641:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516714880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.770679:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574576.770684:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574576.770699:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574576.770704:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574576.770731:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574576.770741:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516714880:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574576.770756:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574576.770759:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574576.770762:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574576.770768:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574576.770775:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574576.770782:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574576.770792:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574576.770801:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574576.770838:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574576.770850:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516714880, portal 10 00000100:00000001:0.0:1788574576.770859:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574576.770867:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788574576.770883:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516714880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.770905:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574576.770909:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574576.770913:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574576.770919:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516714880, offset 0 00000400:00000200:0.0:1788574576.770928:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574576.770938:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574576.770948:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574576.770956:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574576.770961:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574576.770967:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574576.770972:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574576.770978:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547237:547237:256:4294967295] 192.168.202.128@tcp LPNI seq info [547160:547160:8:4294967295] 00000400:00000200:0.0:1788574576.770985:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574576.771005:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574576.771015:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574576.771020:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574576.771027:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574576.771030:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574576.771051:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574576.771053:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574576.771057:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574576.771061:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574576.771065:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574576.771067:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574576.771070:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574576.771074:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516714880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771083:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574576.771086:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516714880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771105:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.771107:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574576.771112:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574576.771116:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574576.771120:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574576.771122:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309936660 00000100:00000001:0.0:1788574576.771125:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309940462 00000100:00000001:0.0:1788574576.771128:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309936660 00000100:00000001:0.0:1788574576.771130:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309940462 00000100:00000001:0.0:1788574576.771132:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574576.771135:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.771137:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574576.771142:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574576/real 1788574576] req@ffff8fc4901a4700 x1875440516714880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574576.771150:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574576.771151:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574576.771153:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574576.771154:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574576.771156:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574576.771157:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574576.771160:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516714880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574576.771165:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516714880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771170:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574576.771171:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771172:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574576.771173:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574576.771175:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516714880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771185:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574576.771188:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516714880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771192:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574576.771194:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771196:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574576.771197:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771198:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574576.771199:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574576.771201:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574576.771203:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574576.771205:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574576.771207:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516714880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574576.771212:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516714880:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574576.771215:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574576.771217:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516715008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771222:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574576.771223:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574576.771224:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574576.771226:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574576.771227:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574576.771229:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715008:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574576.771231:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574576.771233:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574576.771237:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574576.771238:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574576.771241:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574576.771242:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574576.771245:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574576.771246:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574576.771249:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574576.771251:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516715008, portal 10 00000100:00000001:0.0:1788574576.771252:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574576.771253:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788574576.771257:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516715008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771262:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574576.771263:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574576.771265:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574576.771266:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516715008, offset 0 00000400:00000200:0.0:1788574576.771269:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574576.771272:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574576.771274:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574576.771276:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574576.771278:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574576.771280:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574576.771281:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574576.771283:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547238:547238:256:4294967295] 192.168.202.128@tcp LPNI seq info [547161:547161:8:4294967295] 00000400:00000200:0.0:1788574576.771286:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574576.771291:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574576.771294:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574576.771296:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574576.771299:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574576.771301:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574576.771308:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574576.771315:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574576.771317:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574576.771320:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574576.771324:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574576.771326:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574576.771328:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574576.771332:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516715008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771340:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574576.771343:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516715008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771350:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.771352:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574576.771356:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574576.771359:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574576.771363:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574576.771366:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309936667 00000100:00000001:0.0:1788574576.771368:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309940462 00000100:00000001:0.0:1788574576.771370:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309936667 00000100:00000001:0.0:1788574576.771373:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309940462 00000100:00000001:0.0:1788574576.771375:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574576.771377:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.771378:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574576.771381:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574576/real 1788574576] req@ffff8fc4901a7480 x1875440516715008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574576.771387:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574576.771388:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574576.771389:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574576.771391:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574576.771392:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574576.771399:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574576.771401:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516715008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574576.771406:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516715008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771411:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574576.771412:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771413:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574576.771414:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574576.771416:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516715008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771420:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574576.771422:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516715008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771427:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574576.771429:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771430:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574576.771430:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771432:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574576.771434:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574576.771435:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574576.771438:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574576.771440:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574576.771444:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516715008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574576.771451:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715008:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574576.771455:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574576.771464:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516715136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771472:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574576.771475:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574576.771477:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574576.771479:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574576.771482:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574576.771485:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715136:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574576.771488:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574576.771490:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574576.771492:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574576.771493:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574576.771495:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574576.771497:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574576.771500:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574576.771501:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574576.771504:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574576.771506:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516715136, portal 4 00000100:00000001:0.0:1788574576.771508:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574576.771509:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788574576.771513:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516715136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771517:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574576.771519:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574576.771520:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574576.771522:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516715136, offset 0 00000400:00000200:0.0:1788574576.771524:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574576.771527:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574576.771529:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574576.771531:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574576.771533:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574576.771535:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574576.771536:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574576.771542:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547239:547239:256:4294967295] 192.168.202.128@tcp LPNI seq info [547162:547162:8:4294967295] 00000400:00000200:0.0:1788574576.771545:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574576.771550:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574576.771553:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574576.771556:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574576.771558:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574576.771560:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574576.771571:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574576.771572:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574576.771575:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574576.771579:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574576.771582:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574576.771583:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574576.771586:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574576.771590:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516715136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771599:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574576.771602:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516715136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771610:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.771612:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574576.771615:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574576.771617:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574576.771620:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574576.771621:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309936807 00000100:00000001:0.0:1788574576.771623:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309940462 00000100:00000001:0.0:1788574576.771624:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309936807 00000100:00000001:0.0:1788574576.771626:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309940462 00000100:00000001:0.0:1788574576.771628:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574576.771633:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.771635:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574576.771638:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574576/real 1788574576] req@ffff8fc4901a6300 x1875440516715136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574576.771644:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574576.771645:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574576.771646:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574576.771648:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574576.771649:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574576.771650:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574576.771653:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516715136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574576.771662:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516715136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771668:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574576.771670:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771672:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574576.771674:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574576.771677:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516715136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771685:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574576.771689:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516715136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771697:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574576.771700:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771702:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574576.771703:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771705:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574576.771707:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574576.771709:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574576.771728:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574576.771732:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574576.771735:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516715136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574576.771741:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715136:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574576.771744:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771745:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574576.771747:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516714880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771751:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574576.771753:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574576.771754:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574576.771756:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574576.771757:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574576.771758:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574576.771760:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574576.771761:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788574576.771763:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574576.771764:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574576.771765:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788574576.771768:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574576.771769:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771770:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574576.771773:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516715008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771777:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574576.771777:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574576.771778:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574576.771779:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574576.771780:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574576.771781:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574576.771782:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574576.771786:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788574576.771788:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574576.771789:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574576.771790:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788574576.771792:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574576.771793:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771794:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574576.771797:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516715136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771800:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574576.771801:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574576.771802:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574576.771803:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574576.771804:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574576.771805:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574576.771806:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574576.771806:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788574576.771831:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574576.771833:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574576.771834:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788574576.771839:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574576.771840:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771842:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.771846:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574576.771848:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.771851:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574576.771852:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574576.771854:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574576.771858:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516715264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771866:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574576.771867:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574576.771869:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574576.771870:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574576.771872:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574576.771874:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715264:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574576.771882:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574576.771884:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574576.771885:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574576.771886:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574576.771888:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574576.771889:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574576.771892:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574576.771894:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574576.771896:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574576.771899:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516715264, portal 4 00000100:00000001:0.0:1788574576.771900:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574576.771901:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788574576.771905:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516715264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771909:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574576.771911:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574576.771912:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574576.771914:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516715264, offset 0 00000400:00000200:0.0:1788574576.771916:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574576.771919:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574576.771921:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574576.771923:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574576.771925:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574576.771927:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574576.771928:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574576.771931:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547240:547240:256:4294967295] 192.168.202.128@tcp LPNI seq info [547163:547163:8:4294967295] 00000400:00000200:0.0:1788574576.771933:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574576.771937:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574576.771940:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574576.771942:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574576.771945:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574576.771949:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574576.771957:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574576.771958:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574576.771960:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574576.771963:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574576.771965:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574576.771966:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574576.771968:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574576.771971:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516715264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771976:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574576.771978:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516715264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574601 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.771982:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.771983:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574576.771985:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574576.771987:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574576.771990:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574576.771991:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309936810 00000100:00000001:0.0:1788574576.771993:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309940462 00000100:00000001:0.0:1788574576.771994:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309936810 00000100:00000001:0.0:1788574576.771996:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309940462 00000100:00000001:0.0:1788574576.771997:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574576.771998:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.772000:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574576.772002:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574576/real 1788574576] req@ffff8fc4901a4e00 x1875440516715264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574576.772007:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574576.772008:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574576.772023:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574576.772024:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574576.772025:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574576.772027:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574576.772030:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516715264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574576.772035:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516715264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.772039:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574576.772039:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.772041:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574576.772042:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574576.772044:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516715264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.772048:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574576.772051:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516715264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.772055:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574576.772056:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.772057:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574576.772058:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.772059:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574576.772061:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574576.772062:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574576.772064:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574576.772066:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574576.772068:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516715264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574576.772072:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715264:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574576.772078:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.772079:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574576.772082:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516715264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574601 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574576.772086:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574576.772087:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574576.772088:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574576.772090:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574576.772091:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574576.772091:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574576.772093:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574576.772094:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788574576.772095:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574576.772096:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574576.772097:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788574576.772100:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574576.772100:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.772101:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574576.772104:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574576.772105:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574576.772106:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574576.772107:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574577.280932:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574577.280940:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574577.280943:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574577.280951:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574577.280958:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574577.280965:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574577.281002:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574577.281007:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574577.281019:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574577.281081:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574577.281084:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574577.281085:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574577.281088:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574577.281091:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574577.281108:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574577.281110:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574577.281111:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574577.281112:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574577.985023:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574577.985028:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574577.985032:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574578.304903:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574578.304910:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574578.304913:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574578.304920:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574578.304925:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574578.304930:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574578.304933:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574578.304938:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574578.304945:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574578.304965:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574578.304969:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574578.304972:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574578.304975:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574578.304979:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574578.304983:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574578.304986:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574578.304990:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574578.304993:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574579.328921:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574579.328929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574579.328934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574579.328943:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574579.328951:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574579.328957:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574579.328980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574579.328983:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574579.328986:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574580.033121:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574580.033124:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574580.033127:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574580.352936:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574580.352964:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574580.352970:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574580.352979:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574580.352986:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574580.352992:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574580.353002:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574580.353010:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574580.353015:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574580.608929:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309940462, at 4309940480 00000100:00000010:0.0:1788574580.608957:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574581.057035:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574581.057041:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574581.057045:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574581.376876:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574581.376884:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574581.376886:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574581.376896:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574581.376899:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574581.376901:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.376911:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574581.376914:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.376916:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.376957:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574581.376963:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574581.376968:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574581.376977:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574581.376985:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574581.376992:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.376999:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574581.377001:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.377011:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.889206:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574581.889217:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574581.889224:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574581.889244:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516715392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.889274:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574581.889298:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574581.889307:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574581.889314:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574581.889323:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574581.889334:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516715392:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574581.889346:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574581.889355:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574581.889362:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574581.889366:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574581.889374:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574581.889380:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574581.889395:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574581.889407:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574581.889424:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574581.889437:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516715392, portal 25 00000100:00000001:0.0:1788574581.889444:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574581.889449:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788574581.889462:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516715392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.889527:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574581.889536:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574581.889543:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574581.889551:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516715392, offset 0 00000400:00000200:0.0:1788574581.889567:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574581.889580:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574581.889599:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574581.889612:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574581.889623:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574581.889634:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574581.889643:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574581.889654:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547241:547241:256:4294967295] 192.168.202.128@tcp LPNI seq info [547164:547164:8:4294967295] 00000400:00000200:0.0:1788574581.889672:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574581.889714:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574581.889741:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574581.889751:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574581.889768:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574581.889778:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574581.889843:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574581.889847:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574581.889859:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574581.889871:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574581.889882:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574581.889889:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574581.889900:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574581.889912:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516715392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.889935:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574581.889944:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516715392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.889969:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.889974:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574581.889988:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574581.890000:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574581.890007:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309945582 00000100:00000010:0.0:1788574581.890018:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574581.890031:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574581.890055:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309945582 00000100:00000010:0.0:1788574581.890064:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b509c0. 00000100:00000001:0.0:1788574581.890073:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574581.890083:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574581.890092:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.890100:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574581.890110:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574581/real 1788574581] req@ffff8fc4901a4e00 x1875440516715392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574581.890168:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574581.890173:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574581.890178:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574581.890186:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574581.890191:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574581.890199:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574581.890211:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516715392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574581.890237:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516715392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.890259:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574581.890263:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.890271:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574581.890274:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574581.890287:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516715392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.890309:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574581.890324:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516715392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.890338:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574581.890345:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.890350:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574581.890355:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.890361:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574581.890366:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574581.890376:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574581.890390:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574581.890396:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574581.890413:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516715392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574581.890457:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516715392:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574581.890473:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.890482:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574581.890494:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516715392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.890512:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574581.890518:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574581.890524:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574581.890534:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574581.890540:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574581.890545:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574581.890554:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574581.890560:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788574581.890567:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574581.890572:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574581.890575:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788574581.890587:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574581.890592:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.890597:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.890610:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574581.890615:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.890626:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574581.890631:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574581.890635:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574581.890645:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516715520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.890664:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574581.890669:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574581.890676:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574581.890680:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574581.890705:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574581.890715:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715520:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574581.890734:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574581.890741:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574581.890744:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574581.890748:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574581.890753:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574581.890758:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574581.890768:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574581.890773:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574581.890779:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574581.890784:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516715520, portal 10 00000100:00000001:0.0:1788574581.890787:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574581.890789:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788574581.890800:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516715520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.890843:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574581.890850:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574581.890854:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574581.890857:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516715520, offset 0 00000400:00000200:0.0:1788574581.890865:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574581.890871:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574581.890877:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574581.890882:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574581.890886:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574581.890889:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574581.890894:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574581.890897:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547242:547242:256:4294967295] 192.168.202.128@tcp LPNI seq info [547165:547165:8:4294967295] 00000400:00000200:0.0:1788574581.890903:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574581.890911:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574581.890916:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574581.890920:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574581.890924:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574581.890935:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574581.890949:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574581.890951:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574581.890955:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574581.890959:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574581.890963:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574581.890965:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574581.890968:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574581.890972:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516715520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.890981:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574581.890984:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516715520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.890992:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.890994:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574581.890998:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574581.891002:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574581.891006:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574581.891009:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309941780 00000100:00000001:0.0:1788574581.891012:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309945582 00000100:00000001:0.0:1788574581.891015:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309941780 00000100:00000001:0.0:1788574581.891017:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309945582 00000100:00000001:0.0:1788574581.891020:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574581.891022:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.891024:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574581.891029:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574581/real 1788574581] req@ffff8fc4901a6300 x1875440516715520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574581.891037:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574581.891039:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574581.891047:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574581.891051:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574581.891052:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574581.891055:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574581.891059:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516715520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574581.891069:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516715520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891076:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574581.891077:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891079:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574581.891081:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574581.891085:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516715520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891092:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574581.891096:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516715520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891104:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574581.891107:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891109:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574581.891110:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891112:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574581.891115:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574581.891118:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574581.891122:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574581.891125:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574581.891130:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516715520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574581.891139:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715520:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574581.891152:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574581.891157:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516715648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891165:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574581.891166:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574581.891169:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574581.891171:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574581.891173:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574581.891177:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715648:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574581.891180:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574581.891183:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574581.891185:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574581.891187:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574581.891191:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574581.891194:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574581.891198:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574581.891200:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574581.891205:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574581.891208:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516715648, portal 10 00000100:00000001:0.0:1788574581.891210:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574581.891212:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788574581.891219:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516715648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891227:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574581.891229:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574581.891231:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574581.891234:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516715648, offset 0 00000400:00000200:0.0:1788574581.891238:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574581.891242:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574581.891246:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574581.891250:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574581.891258:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574581.891261:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574581.891263:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574581.891267:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547243:547243:256:4294967295] 192.168.202.128@tcp LPNI seq info [547166:547166:8:4294967295] 00000400:00000200:0.0:1788574581.891272:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574581.891279:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574581.891285:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574581.891288:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574581.891292:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574581.891295:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574581.891308:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574581.891310:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574581.891313:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574581.891317:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574581.891320:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574581.891322:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574581.891325:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574581.891330:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516715648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891339:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574581.891342:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516715648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891350:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.891352:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574581.891355:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574581.891359:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574581.891363:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574581.891365:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309941787 00000100:00000001:0.0:1788574581.891368:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309945582 00000100:00000001:0.0:1788574581.891371:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309941787 00000100:00000001:0.0:1788574581.891379:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309945582 00000100:00000001:0.0:1788574581.891382:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574581.891384:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.891387:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574581.891392:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574581/real 1788574581] req@ffff8fc4901a7480 x1875440516715648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574581.891400:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574581.891402:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574581.891404:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574581.891406:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574581.891408:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574581.891410:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574581.891413:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516715648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574581.891421:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516715648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891427:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574581.891429:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891431:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574581.891432:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574581.891435:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516715648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891442:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574581.891446:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516715648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891454:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574581.891456:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891458:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574581.891459:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891466:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574581.891469:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574581.891472:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574581.891476:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574581.891478:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574581.891483:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516715648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574581.891492:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715648:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574581.891496:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574581.891500:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516715776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891508:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574581.891510:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574581.891512:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574581.891514:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574581.891517:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574581.891520:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715776:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574581.891524:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574581.891526:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574581.891528:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574581.891530:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574581.891534:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574581.891536:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574581.891541:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574581.891543:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574581.891548:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574581.891551:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516715776, portal 4 00000100:00000001:0.0:1788574581.891554:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574581.891556:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788574581.891562:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516715776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891575:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574581.891578:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574581.891580:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574581.891582:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516715776, offset 0 00000400:00000200:0.0:1788574581.891585:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574581.891588:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574581.891591:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574581.891593:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574581.891595:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574581.891597:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574581.891598:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574581.891600:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547244:547244:256:4294967295] 192.168.202.128@tcp LPNI seq info [547167:547167:8:4294967295] 00000400:00000200:0.0:1788574581.891603:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574581.891608:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574581.891611:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574581.891614:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574581.891617:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574581.891619:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574581.891626:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574581.891628:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574581.891630:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574581.891633:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574581.891635:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574581.891636:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574581.891638:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574581.891641:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516715776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891647:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574581.891649:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516715776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891662:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.891663:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574581.891666:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574581.891668:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574581.891671:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574581.891672:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309941927 00000100:00000001:0.0:1788574581.891674:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309945582 00000100:00000001:0.0:1788574581.891676:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309941927 00000100:00000001:0.0:1788574581.891678:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309945582 00000100:00000001:0.0:1788574581.891679:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574581.891680:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.891681:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574581.891684:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574581/real 1788574581] req@ffff8fc4901a4700 x1875440516715776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574581.891704:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574581.891706:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574581.891708:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574581.891710:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574581.891712:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574581.891713:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574581.891716:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516715776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574581.891722:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516715776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891726:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574581.891727:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891729:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574581.891729:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574581.891732:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516715776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891741:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574581.891744:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516715776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891749:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574581.891751:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891752:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574581.891753:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891754:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574581.891755:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574581.891757:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574581.891759:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574581.891760:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574581.891763:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516715776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574581.891768:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715776:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574581.891771:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891772:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574581.891775:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516715520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891779:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574581.891780:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574581.891782:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574581.891783:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574581.891785:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574581.891786:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574581.891788:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574581.891789:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788574581.891795:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574581.891797:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574581.891798:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788574581.891802:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574581.891803:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891805:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574581.891824:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516715648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891829:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574581.891830:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574581.891832:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574581.891834:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574581.891835:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574581.891835:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574581.891837:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574581.891838:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788574581.891840:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574581.891841:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574581.891842:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788574581.891845:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574581.891846:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891847:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574581.891850:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516715776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891854:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574581.891855:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574581.891856:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574581.891857:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574581.891857:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574581.891858:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574581.891859:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574581.891860:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788574581.891861:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574581.891862:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574581.891863:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788574581.891864:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574581.891868:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891870:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.891873:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574581.891874:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.891875:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574581.891877:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574581.891878:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574581.891881:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516715904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891886:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574581.891887:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574581.891889:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574581.891891:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574581.891892:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574581.891894:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715904:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574581.891896:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574581.891898:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574581.891900:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574581.891901:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574581.891903:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574581.891904:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574581.891907:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574581.891909:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574581.891911:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574581.891914:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516715904, portal 4 00000100:00000001:0.0:1788574581.891915:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574581.891916:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788574581.891921:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516715904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891925:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574581.891927:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574581.891928:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574581.891930:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516715904, offset 0 00000400:00000200:0.0:1788574581.891932:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574581.891939:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574581.891942:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574581.891944:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574581.891946:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574581.891948:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574581.891949:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574581.891952:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547245:547245:256:4294967295] 192.168.202.128@tcp LPNI seq info [547168:547168:8:4294967295] 00000400:00000200:0.0:1788574581.891955:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574581.891959:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574581.891962:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574581.891965:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574581.891967:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574581.891969:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574581.891977:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574581.891979:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574581.891981:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574581.891983:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574581.891985:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574581.891986:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574581.891988:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574581.891991:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516715904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.891996:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574581.891998:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516715904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574606 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.892003:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.892004:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574581.892006:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574581.892008:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574581.892010:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574581.892015:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309941930 00000100:00000001:0.0:1788574581.892017:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309945582 00000100:00000001:0.0:1788574581.892019:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309941930 00000100:00000001:0.0:1788574581.892020:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309945582 00000100:00000001:0.0:1788574581.892022:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574581.892023:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.892025:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574581.892027:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574581/real 1788574581] req@ffff8fc4901a5880 x1875440516715904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574581.892032:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574581.892033:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574581.892035:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574581.892036:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574581.892038:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574581.892039:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574581.892041:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516715904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574581.892047:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516715904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.892051:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574581.892052:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.892053:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574581.892054:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574581.892056:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516715904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.892060:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574581.892062:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516715904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.892066:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574581.892070:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.892072:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574581.892072:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.892074:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574581.892076:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574581.892077:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574581.892080:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574581.892081:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574581.892083:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516715904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574581.892089:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516715904:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574581.892091:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.892093:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574581.892095:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516715904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574606 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574581.892098:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574581.892099:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574581.892100:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574581.892102:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574581.892102:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574581.892103:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574581.892104:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574581.892105:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788574581.892107:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574581.892107:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574581.892108:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788574581.892111:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574581.892112:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.892113:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574581.892117:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574581.892118:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574581.892120:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574581.892126:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574582.400966:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574582.400971:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574582.400977:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574582.400987:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574582.400998:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574582.401006:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574582.401032:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574582.401036:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574582.401040:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574583.104983:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574583.104986:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574583.104989:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574583.424947:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574583.424963:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574583.424966:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574583.424970:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574583.424976:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574583.424980:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574583.424984:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574583.424987:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574583.424989:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574584.448909:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574584.448915:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574584.448919:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574584.448927:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574584.448933:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574584.448939:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574584.448944:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574584.448948:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574584.448953:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574585.152965:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574585.152970:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574585.152973:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574585.472901:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574585.472911:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574585.472914:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574585.472923:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574585.472950:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574585.472955:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574585.472962:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574585.472964:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574585.472969:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574585.728949:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309945582, at 4309945600 00000100:00000010:0.0:1788574585.728969:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574586.177078:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574586.177085:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574586.177090:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574586.496913:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574586.496921:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574586.496926:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574586.496942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574586.496947:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574586.496953:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574586.496966:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574586.496973:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574586.496985:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.009251:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574587.009256:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574587.009261:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574587.009270:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7b80 x1875440516716032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.009298:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574587.009304:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574587.009311:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574587.009315:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574587.009323:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574587.009330:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516716032:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574587.009337:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574587.009342:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574587.009346:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574587.009350:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574587.009355:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574587.009368:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574587.009377:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574587.009391:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574587.009399:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574587.009411:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516716032, portal 25 00000100:00000001:0.0:1788574587.009417:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574587.009421:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335936 : -123400104215680 : ffff8fc4ac9f7b80) 00000100:00000040:0.0:1788574587.009434:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7b80 x1875440516716032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.009457:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574587.009465:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574587.009470:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574587.009477:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516716032, offset 0 00000400:00000200:0.0:1788574587.009493:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574587.009507:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574587.009518:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574587.009527:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574587.009533:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574587.009539:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574587.009544:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574587.009552:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547246:547246:256:4294967295] 192.168.202.128@tcp LPNI seq info [547169:547169:8:4294967295] 00000400:00000200:0.0:1788574587.009561:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574587.009576:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574587.009584:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574587.009591:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574587.009603:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574587.009609:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574587.009636:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574587.009639:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574587.009644:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574587.009652:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574587.009684:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574587.009688:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574587.009697:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574587.009713:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7b80 x1875440516716032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.009740:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574587.009760:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7b80 x1875440516716032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.009784:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.009790:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574587.009801:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574587.009850:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574587.009856:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309950702 00000100:00000010:0.0:1788574587.009865:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574587.009883:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574587.009894:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309950702 00000100:00000010:0.0:1788574587.009902:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50540. 00000100:00000001:0.0:1788574587.009911:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574587.009918:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574587.009925:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.009930:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574587.009940:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574587/real 1788574587] req@ffff8fc4ac9f7b80 x1875440516716032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574587.009961:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574587.009964:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574587.009975:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574587.009982:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574587.009986:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574587.009992:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574587.009997:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7b80 x1875440516716032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574587.010016:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7b80 x1875440516716032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010046:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574587.010052:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010059:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574587.010062:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574587.010073:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7b80 x1875440516716032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010090:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574587.010099:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7b80 x1875440516716032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010109:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574587.010112:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010115:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574587.010118:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010123:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574587.010125:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574587.010130:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574587.010137:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574587.010142:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574587.010153:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7b80 x1875440516716032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574587.010169:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516716032:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574587.010177:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010179:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574587.010183:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7b80 x1875440516716032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010190:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574587.010192:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574587.010199:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574587.010201:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574587.010202:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574587.010203:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574587.010205:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574587.010207:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35800. 02000000:00000001:0.0:1788574587.010210:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574587.010212:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574587.010213:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7b80. 00000100:00000001:0.0:1788574587.010215:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574587.010217:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010218:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010224:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574587.010225:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.010226:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574587.010228:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574587.010229:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574587.010232:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4a80 x1875440516716160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010237:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574587.010238:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574587.010239:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574587.010241:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574587.010242:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574587.010244:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516716160:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574587.010246:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574587.010248:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574587.010249:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574587.010250:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574587.010252:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574587.010253:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574587.010257:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574587.010259:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574587.010261:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574587.010264:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516716160, portal 10 00000100:00000001:0.0:1788574587.010268:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574587.010269:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605323392 : -123400104228224 : ffff8fc4ac9f4a80) 00000100:00000040:0.0:1788574587.010275:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4a80 x1875440516716160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010282:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574587.010285:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574587.010287:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574587.010290:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516716160, offset 0 00000400:00000200:0.0:1788574587.010295:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574587.010299:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574587.010303:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574587.010306:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574587.010309:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574587.010312:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574587.010315:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574587.010318:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547247:547247:256:4294967295] 192.168.202.128@tcp LPNI seq info [547170:547170:8:4294967295] 00000400:00000200:0.0:1788574587.010323:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574587.010329:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574587.010332:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574587.010335:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574587.010338:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574587.010340:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574587.010347:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574587.010349:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574587.010351:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574587.010353:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574587.010356:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574587.010357:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574587.010359:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574587.010361:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4a80 x1875440516716160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010376:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574587.010378:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4a80 x1875440516716160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010382:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.010383:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574587.010386:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574587.010389:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574587.010392:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574587.010393:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309946899 00000100:00000001:0.0:1788574587.010396:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309950702 00000100:00000001:0.0:1788574587.010397:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309946899 00000100:00000001:0.0:1788574587.010399:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309950702 00000100:00000001:0.0:1788574587.010400:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574587.010402:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.010403:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574587.010405:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574587/real 1788574587] req@ffff8fc4ac9f4a80 x1875440516716160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574587.010410:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574587.010411:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574587.010413:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574587.010414:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574587.010416:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574587.010417:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574587.010419:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4a80 x1875440516716160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574587.010424:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4a80 x1875440516716160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010428:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574587.010432:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010433:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574587.010434:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574587.010437:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4a80 x1875440516716160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010442:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574587.010444:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4a80 x1875440516716160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010449:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574587.010450:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010451:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574587.010452:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010453:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574587.010455:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574587.010457:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574587.010458:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574587.010460:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574587.010462:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4a80 x1875440516716160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574587.010467:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516716160:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574587.010470:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574587.010472:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5880 x1875440516716288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010476:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574587.010477:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574587.010479:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574587.010480:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574587.010481:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574587.010487:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516716288:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574587.010490:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574587.010491:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574587.010492:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574587.010494:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574587.010496:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574587.010497:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574587.010499:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574587.010501:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574587.010503:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574587.010505:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516716288, portal 10 00000100:00000001:0.0:1788574587.010506:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574587.010507:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605326976 : -123400104224640 : ffff8fc4ac9f5880) 00000100:00000040:0.0:1788574587.010511:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5880 x1875440516716288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010515:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574587.010517:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574587.010518:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574587.010520:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516716288, offset 0 00000400:00000200:0.0:1788574587.010522:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574587.010525:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574587.010527:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574587.010529:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574587.010531:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574587.010532:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574587.010534:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574587.010536:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547248:547248:256:4294967295] 192.168.202.128@tcp LPNI seq info [547171:547171:8:4294967295] 00000400:00000200:0.0:1788574587.010539:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574587.010544:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574587.010547:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574587.010549:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574587.010555:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574587.010557:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574587.010564:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574587.010565:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574587.010567:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574587.010570:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574587.010572:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574587.010573:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574587.010575:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574587.010577:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5880 x1875440516716288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010582:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574587.010585:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5880 x1875440516716288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010589:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.010590:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574587.010592:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574587.010594:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574587.010596:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574587.010598:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309946906 00000100:00000001:0.0:1788574587.010599:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309950702 00000100:00000001:0.0:1788574587.010601:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309946906 00000100:00000001:0.0:1788574587.010602:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309950702 00000100:00000001:0.0:1788574587.010604:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574587.010605:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.010606:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574587.010608:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574587/real 1788574587] req@ffff8fc4ac9f5880 x1875440516716288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574587.010613:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574587.010617:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574587.010618:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574587.010619:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574587.010620:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574587.010622:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574587.010625:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5880 x1875440516716288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574587.010630:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5880 x1875440516716288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010634:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574587.010635:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010636:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574587.010637:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574587.010639:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5880 x1875440516716288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010643:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574587.010645:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5880 x1875440516716288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010649:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574587.010651:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010652:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574587.010653:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010654:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574587.010674:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574587.010676:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574587.010679:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574587.010681:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574587.010686:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5880 x1875440516716288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574587.010707:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516716288:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574587.010711:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574587.010715:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6d80 x1875440516716416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010722:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574587.010724:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574587.010726:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574587.010728:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574587.010731:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574587.010734:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516716416:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574587.010737:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574587.010740:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574587.010741:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574587.010743:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574587.010746:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574587.010749:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574587.010753:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574587.010756:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574587.010760:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574587.010763:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516716416, portal 4 00000100:00000001:0.0:1788574587.010765:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574587.010767:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605332352 : -123400104219264 : ffff8fc4ac9f6d80) 00000100:00000040:0.0:1788574587.010773:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6d80 x1875440516716416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010780:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574587.010783:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574587.010785:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574587.010788:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516716416, offset 0 00000400:00000200:0.0:1788574587.010792:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574587.010803:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574587.010822:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574587.010826:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574587.010828:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574587.010830:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574587.010833:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574587.010836:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547249:547249:256:4294967295] 192.168.202.128@tcp LPNI seq info [547172:547172:8:4294967295] 00000400:00000200:0.0:1788574587.010841:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574587.010848:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574587.010853:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574587.010857:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574587.010862:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574587.010865:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574587.010881:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574587.010883:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574587.010886:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574587.010889:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574587.010892:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574587.010893:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574587.010895:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574587.010898:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6d80 x1875440516716416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010904:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574587.010906:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6d80 x1875440516716416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010910:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.010911:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574587.010914:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574587.010916:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574587.010918:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574587.010923:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309947047 00000100:00000001:0.0:1788574587.010925:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309950702 00000100:00000001:0.0:1788574587.010927:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309947047 00000100:00000001:0.0:1788574587.010928:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309950702 00000100:00000001:0.0:1788574587.010930:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574587.010931:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.010932:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574587.010935:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574587/real 1788574587] req@ffff8fc4ac9f6d80 x1875440516716416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574587.010940:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574587.010941:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574587.010942:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574587.010943:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574587.010944:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574587.010946:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574587.010947:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6d80 x1875440516716416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574587.010953:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6d80 x1875440516716416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010957:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574587.010958:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010959:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574587.010960:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574587.010962:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6d80 x1875440516716416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010966:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574587.010969:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6d80 x1875440516716416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.010973:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574587.010979:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010980:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574587.010981:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.010982:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574587.010983:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574587.010985:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574587.010987:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574587.010989:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574587.010991:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6d80 x1875440516716416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574587.010997:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516716416:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574587.010999:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574587.011001:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5c00 x1875440516716544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.011005:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574587.011006:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574587.011008:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574587.011010:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574587.011012:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574587.011015:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516716544:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574587.011017:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574587.011019:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574587.011020:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574587.011022:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574587.011026:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574587.011028:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574587.011031:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574587.011034:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574587.011038:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574587.011041:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516716544, portal 4 00000100:00000001:0.0:1788574587.011048:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574587.011049:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605327872 : -123400104223744 : ffff8fc4ac9f5c00) 00000100:00000040:0.0:1788574587.011056:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5c00 x1875440516716544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.011063:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574587.011065:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574587.011068:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574587.011070:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516716544, offset 0 00000400:00000200:0.0:1788574587.011073:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574587.011076:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574587.011078:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574587.011080:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574587.011082:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574587.011084:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574587.011085:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574587.011087:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547250:547250:256:4294967295] 192.168.202.128@tcp LPNI seq info [547173:547173:8:4294967295] 00000400:00000200:0.0:1788574587.011090:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574587.011095:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574587.011098:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574587.011100:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574587.011103:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574587.011105:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574587.011112:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574587.011113:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574587.011115:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574587.011118:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574587.011120:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574587.011121:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574587.011123:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574587.011126:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5c00 x1875440516716544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.011146:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574587.011151:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5c00 x1875440516716544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574612 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.011158:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.011159:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574587.011164:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574587.011168:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574587.011171:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574587.011174:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309947049 00000100:00000001:0.0:1788574587.011176:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309950702 00000100:00000001:0.0:1788574587.011179:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309947049 00000100:00000001:0.0:1788574587.011181:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309950702 00000100:00000001:0.0:1788574587.011183:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574587.011186:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.011188:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574587.011192:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574587/real 1788574587] req@ffff8fc4ac9f5c00 x1875440516716544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574587.011201:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574587.011203:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574587.011204:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574587.011207:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574587.011209:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574587.011212:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574587.011216:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5c00 x1875440516716544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574587.011225:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5c00 x1875440516716544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.011233:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574587.011240:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.011242:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574587.011244:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574587.011249:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5c00 x1875440516716544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.011257:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574587.011261:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5c00 x1875440516716544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.011269:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574587.011272:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.011273:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574587.011275:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.011277:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574587.011278:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574587.011281:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574587.011285:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574587.011287:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574587.011292:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5c00 x1875440516716544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574587.011301:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516716544:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574587.011305:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.011307:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574587.011310:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4a80 x1875440516716160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.011317:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574587.011319:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574587.011321:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574587.011324:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574587.011325:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574587.011333:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574587.011335:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574587.011338:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788574587.011341:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574587.011342:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574587.011344:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4a80. 00000100:00000001:0.0:1788574587.011347:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574587.011348:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.011350:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574587.011354:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5880 x1875440516716288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.011359:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574587.011360:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574587.011361:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574587.011363:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574587.011364:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574587.011364:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574587.011365:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574587.011366:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788574587.011368:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574587.011368:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574587.011369:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5880. 00000100:00000001:0.0:1788574587.011371:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574587.011372:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.011373:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574587.011375:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6d80 x1875440516716416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.011379:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574587.011380:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574587.011380:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574587.011382:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574587.011382:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574587.011383:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574587.011384:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574587.011385:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788574587.011386:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574587.011391:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574587.011392:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6d80. 00000100:00000001:0.0:1788574587.011394:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574587.011395:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.011396:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574587.011398:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5c00 x1875440516716544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574612 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574587.011402:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574587.011403:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574587.011404:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574587.011406:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574587.011407:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574587.011408:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574587.011409:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574587.011410:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788574587.011412:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574587.011412:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574587.011413:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5c00. 00000100:00000001:0.0:1788574587.011414:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574587.011415:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.011416:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.011419:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574587.011420:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.011421:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574587.011422:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574587.520999:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574587.521007:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574587.521011:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574587.521024:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574587.521029:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574587.521037:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.521063:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574587.521067:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574587.521072:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574588.225060:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574588.225067:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574588.225071:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574588.544936:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574588.544947:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574588.544952:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574588.544959:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574588.544968:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574588.544974:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574588.544981:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574588.544984:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574588.544988:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574589.568960:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574589.568970:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574589.568973:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574589.568980:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574589.568988:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574589.568996:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574589.569001:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574589.569006:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574589.569013:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574589.569055:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574589.569062:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574589.569071:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574589.569081:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574589.569097:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574589.569108:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574589.569112:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574589.569113:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574589.569115:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574590.273110:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574590.273116:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574590.273120:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574590.592981:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574590.592988:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574590.592994:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574590.593014:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574590.593031:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574590.593035:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574590.593039:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574590.593045:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574590.593065:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574590.848947:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309950702, at 4309950720 00000100:00000010:0.0:1788574590.848969:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574591.296994:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574591.297000:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574591.297004:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574591.616952:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574591.616958:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574591.616961:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574591.616969:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574591.616976:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574591.616985:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574591.616990:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574591.616994:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574591.616999:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574591.617019:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574591.617024:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574591.617026:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574591.617030:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574591.617032:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574591.617034:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574591.617040:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574591.617042:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574591.617045:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.129273:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574592.129280:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574592.129285:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574592.129295:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516716672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.129312:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574592.129314:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574592.129320:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574592.129325:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574592.129328:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574592.129335:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516716672:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574592.129353:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574592.129359:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574592.129363:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574592.129368:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574592.129374:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574592.129378:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574592.129385:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574592.129390:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574592.129400:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574592.129410:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516716672, portal 25 00000100:00000001:0.0:1788574592.129414:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574592.129417:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788574592.129425:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516716672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.129448:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574592.129454:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574592.129457:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574592.129461:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516716672, offset 0 00000400:00000200:0.0:1788574592.129470:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574592.129480:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574592.129490:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574592.129500:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574592.129507:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574592.129514:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574592.129520:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574592.129524:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547251:547251:256:4294967295] 192.168.202.128@tcp LPNI seq info [547174:547174:8:4294967295] 00000400:00000200:0.0:1788574592.129532:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574592.129549:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574592.129562:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574592.129571:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574592.129581:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574592.129594:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574592.129619:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574592.129623:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574592.129648:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574592.129657:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574592.129665:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574592.129668:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574592.129676:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574592.129685:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516716672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.129700:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574592.129706:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516716672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.129737:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.129742:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574592.129752:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574592.129763:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574592.129768:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309955821 00000100:00000010:0.0:1788574592.129774:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574592.129787:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574592.129799:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309955821 00000100:00000010:0.0:1788574592.129847:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b502a0. 00000100:00000001:0.0:1788574592.129856:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574592.129871:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574592.129879:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.129886:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574592.129898:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574592/real 1788574592] req@ffff8fc4ac821880 x1875440516716672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574592.129913:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574592.129917:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574592.129927:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574592.129933:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574592.129936:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574592.129942:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574592.129948:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516716672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574592.129965:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516716672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.129979:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574592.129983:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.129986:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574592.129990:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574592.129997:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516716672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130012:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574592.130021:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516716672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130036:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574592.130039:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.130043:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574592.130048:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.130053:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574592.130056:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574592.130059:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574592.130064:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574592.130067:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574592.130075:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516716672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574592.130094:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516716672:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574592.130113:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.130119:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574592.130128:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516716672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130152:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574592.130158:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574592.130164:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574592.130171:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574592.130175:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574592.130178:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574592.130183:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574592.130188:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574592.130195:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574592.130198:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574592.130199:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788574592.130203:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574592.130205:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.130208:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.130219:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574592.130222:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.130224:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574592.130226:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574592.130227:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574592.130232:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516716800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130237:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574592.130238:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574592.130240:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574592.130241:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574592.130243:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574592.130245:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516716800:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574592.130247:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574592.130249:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574592.130250:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574592.130257:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574592.130259:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574592.130260:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574592.130264:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574592.130267:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574592.130269:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574592.130272:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516716800, portal 10 00000100:00000001:0.0:1788574592.130273:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574592.130274:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788574592.130280:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516716800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130285:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574592.130287:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574592.130288:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574592.130290:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516716800, offset 0 00000400:00000200:0.0:1788574592.130292:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574592.130295:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574592.130299:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574592.130303:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574592.130305:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574592.130308:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574592.130311:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574592.130315:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547252:547252:256:4294967295] 192.168.202.128@tcp LPNI seq info [547175:547175:8:4294967295] 00000400:00000200:0.0:1788574592.130319:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574592.130326:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574592.130331:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574592.130335:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574592.130340:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574592.130343:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574592.130354:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574592.130355:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574592.130362:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574592.130365:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574592.130368:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574592.130369:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574592.130371:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574592.130374:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516716800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130380:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574592.130382:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516716800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130387:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.130388:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574592.130391:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574592.130394:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574592.130397:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574592.130398:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309952019 00000100:00000001:0.0:1788574592.130401:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309955821 00000100:00000001:0.0:1788574592.130403:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309952019 00000100:00000001:0.0:1788574592.130404:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309955821 00000100:00000001:0.0:1788574592.130406:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574592.130407:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.130409:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574592.130411:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574592/real 1788574592] req@ffff8fc4ac820e00 x1875440516716800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574592.130416:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574592.130417:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574592.130418:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574592.130420:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574592.130421:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574592.130422:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574592.130430:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516716800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574592.130436:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516716800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130440:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574592.130441:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.130442:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574592.130443:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574592.130445:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516716800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130450:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574592.130453:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516716800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130458:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574592.130460:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.130461:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574592.130461:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.130463:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574592.130464:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574592.130465:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574592.130467:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574592.130469:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574592.130471:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516716800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574592.130476:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516716800:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574592.130480:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574592.130483:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516716928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130502:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574592.130504:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574592.130506:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574592.130508:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574592.130510:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574592.130514:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516716928:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574592.130518:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574592.130521:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574592.130522:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574592.130524:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574592.130528:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574592.130530:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574592.130534:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574592.130537:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574592.130541:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574592.130544:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516716928, portal 10 00000100:00000001:0.0:1788574592.130546:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574592.130548:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788574592.130555:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516716928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130562:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574592.130565:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574592.130567:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574592.130570:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516716928, offset 0 00000400:00000200:0.0:1788574592.130574:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574592.130579:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574592.130584:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574592.130587:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574592.130590:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574592.130592:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574592.130594:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574592.130603:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547253:547253:256:4294967295] 192.168.202.128@tcp LPNI seq info [547176:547176:8:4294967295] 00000400:00000200:0.0:1788574592.130607:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574592.130613:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574592.130619:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574592.130623:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574592.130640:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574592.130644:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574592.130654:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574592.130657:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574592.130660:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574592.130663:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574592.130667:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574592.130669:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574592.130672:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574592.130676:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516716928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130684:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574592.130687:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516716928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130693:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.130694:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574592.130697:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574592.130700:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574592.130704:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574592.130705:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309952026 00000100:00000001:0.0:1788574592.130708:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309955821 00000100:00000001:0.0:1788574592.130710:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309952026 00000100:00000001:0.0:1788574592.130712:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309955821 00000100:00000001:0.0:1788574592.130714:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574592.130722:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.130723:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574592.130728:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574592/real 1788574592] req@ffff8fc4ac823800 x1875440516716928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574592.130737:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574592.130738:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574592.130741:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574592.130744:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574592.130745:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574592.130748:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574592.130751:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516716928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574592.130760:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516716928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130767:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574592.130769:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.130771:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574592.130772:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574592.130776:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516716928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130783:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574592.130788:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516716928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130795:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574592.130798:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.130800:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574592.130801:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.130803:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574592.130806:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574592.130829:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574592.130839:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574592.130841:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574592.130846:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516716928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574592.130855:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516716928:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574592.130859:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574592.130863:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516717056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130869:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574592.130871:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574592.130873:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574592.130875:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574592.130877:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574592.130880:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516717056:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574592.130883:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574592.130885:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574592.130887:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574592.130889:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574592.130892:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574592.130894:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574592.130898:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574592.130902:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574592.130906:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574592.130909:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516717056, portal 4 00000100:00000001:0.0:1788574592.130911:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574592.130913:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788574592.130919:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516717056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.130926:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574592.130936:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574592.130939:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574592.130942:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516717056, offset 0 00000400:00000200:0.0:1788574592.130946:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574592.130950:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574592.130954:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574592.130956:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574592.130959:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574592.130961:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574592.130963:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574592.130967:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547254:547254:256:4294967295] 192.168.202.128@tcp LPNI seq info [547177:547177:8:4294967295] 00000400:00000200:0.0:1788574592.130971:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574592.130977:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574592.130981:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574592.130985:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574592.130988:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574592.130991:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574592.131001:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574592.131003:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574592.131006:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574592.131009:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574592.131012:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574592.131014:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574592.131017:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574592.131020:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516717056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131028:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574592.131032:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516717056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131039:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.131040:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574592.131049:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574592.131052:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574592.131056:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574592.131059:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309952167 00000100:00000001:0.0:1788574592.131061:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309955821 00000100:00000001:0.0:1788574592.131064:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309952167 00000100:00000001:0.0:1788574592.131067:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309955821 00000100:00000001:0.0:1788574592.131069:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574592.131071:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.131073:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574592.131078:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574592/real 1788574592] req@ffff8fc4ac820a80 x1875440516717056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574592.131087:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574592.131089:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574592.131091:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574592.131093:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574592.131095:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574592.131097:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574592.131100:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516717056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574592.131108:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516717056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131115:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574592.131116:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.131118:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574592.131120:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574592.131124:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516717056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131131:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574592.131136:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516717056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131152:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574592.131155:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.131157:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574592.131158:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.131161:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574592.131163:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574592.131166:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574592.131170:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574592.131173:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574592.131178:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516717056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574592.131186:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516717056:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574592.131191:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574592.131195:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516717184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131202:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574592.131203:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574592.131205:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574592.131207:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574592.131210:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574592.131212:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516717184:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574592.131216:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574592.131218:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574592.131220:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574592.131221:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574592.131226:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574592.131228:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574592.131232:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574592.131241:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574592.131245:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574592.131249:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516717184, portal 4 00000100:00000001:0.0:1788574592.131251:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574592.131252:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788574592.131259:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516717184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131266:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574592.131268:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574592.131271:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574592.131274:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516717184, offset 0 00000400:00000200:0.0:1788574592.131278:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574592.131282:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574592.131286:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574592.131289:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574592.131291:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574592.131294:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574592.131296:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574592.131300:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547255:547255:256:4294967295] 192.168.202.128@tcp LPNI seq info [547178:547178:8:4294967295] 00000400:00000200:0.0:1788574592.131304:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574592.131311:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574592.131315:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574592.131319:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574592.131323:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574592.131326:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574592.131338:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574592.131340:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574592.131344:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574592.131347:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574592.131351:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574592.131353:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574592.131361:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574592.131366:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516717184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131374:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574592.131378:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516717184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574617 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131384:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.131386:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574592.131390:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574592.131393:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574592.131397:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574592.131399:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309952169 00000100:00000001:0.0:1788574592.131402:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309955821 00000100:00000001:0.0:1788574592.131404:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309952169 00000100:00000001:0.0:1788574592.131407:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309955821 00000100:00000001:0.0:1788574592.131410:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574592.131412:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.131414:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574592.131419:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574592/real 1788574592] req@ffff8fc4ac821c00 x1875440516717184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574592.131427:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574592.131429:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574592.131431:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574592.131433:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574592.131435:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574592.131437:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574592.131441:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516717184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574592.131450:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516717184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131468:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574592.131470:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.131473:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574592.131474:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574592.131479:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516717184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131486:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574592.131490:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516717184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131497:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574592.131500:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.131502:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574592.131503:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.131505:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574592.131508:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574592.131511:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574592.131514:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574592.131516:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574592.131521:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516717184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574592.131529:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516717184:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574592.131534:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.131536:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574592.131539:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516716800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131546:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574592.131547:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574592.131555:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574592.131557:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574592.131559:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574592.131561:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574592.131565:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574592.131567:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788574592.131570:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574592.131571:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574592.131573:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788574592.131576:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574592.131578:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.131580:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574592.131584:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516716928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131592:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574592.131594:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574592.131596:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574592.131598:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574592.131600:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574592.131601:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574592.131604:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574592.131606:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788574592.131608:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574592.131609:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574592.131611:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788574592.131614:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574592.131615:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.131617:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574592.131621:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516717056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131644:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574592.131646:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574592.131647:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574592.131650:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574592.131651:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574592.131653:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574592.131661:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574592.131663:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788574592.131666:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574592.131667:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574592.131669:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788574592.131671:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574592.131673:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.131675:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574592.131680:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516717184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574617 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574592.131687:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574592.131689:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574592.131690:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574592.131694:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574592.131695:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574592.131696:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574592.131699:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574592.131700:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788574592.131703:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574592.131704:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574592.131706:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788574592.131708:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574592.131709:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.131711:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.131716:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574592.131717:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.131719:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574592.131721:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.640908:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574592.640916:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574592.640921:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574592.640926:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574592.640932:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574592.640937:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.640942:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574592.640948:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.640953:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574592.640998:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574592.641002:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574592.641003:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574592.641006:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574592.641009:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574592.641011:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.641014:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574592.641016:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574592.641019:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574593.345008:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574593.345017:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574593.345020:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574593.664957:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574593.664969:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574593.664974:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574593.664981:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574593.664989:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574593.664997:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574593.665019:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574593.665021:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574593.665023:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574593.665039:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574593.665043:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574593.665045:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574593.665048:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574593.665051:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574593.665053:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574593.665056:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574593.665058:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574593.665060:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.688946:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574594.688956:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574594.688961:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574594.688968:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574594.688974:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574594.688980:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574594.688986:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574594.688992:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574594.689005:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574594.689022:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574594.689028:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574594.689032:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574594.689039:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574594.689043:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574594.689046:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574594.689048:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574594.689049:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574594.689051:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574595.393061:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574595.393068:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574595.393074:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574595.968931:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309955821, at 4309955840 00000100:00000010:0.0:1788574595.968948:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574596.416985:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574596.416988:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574596.416990:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574596.736967:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574596.736976:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574596.736978:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574596.736988:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574596.736998:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574596.737005:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574596.737011:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574596.737015:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574596.737024:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574597.248992:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574597.249004:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574597.249008:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574597.249012:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574597.249017:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574597.249024:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15284 00000400:00000200:0.0:1788574597.249035:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574597.249043:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574597.249065:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574597.249069:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249073:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574597.249076:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249079:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574597.249084:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574597.249086:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574597.249089:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574597.249092:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574597.249095:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574597.249097:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574597.249101:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249103:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574597.249105:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249108:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574597.249111:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574597.249115:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249117:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574597.249121:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574597.249124:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574597.249128:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.249130:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574597.249133:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574597.249141:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574597.249144:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574597.249149:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574597.249150:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.249153:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574597.249156:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574597.249161:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574597.249169:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574597.249176:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574597.249192:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516717312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574597.249204:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574597.249208:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574597.249213:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516717312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574597.249240:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574597.249244:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574597.249248:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574597.249249:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574597.249251:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574597.249254:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574597.249257:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15284 00000400:00000200:0.0:1788574597.249262:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574597.249266:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574597.249269:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574597.249273:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249275:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574597.249278:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249280:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574597.249284:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574597.249287:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574597.249292:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574597.249296:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574597.249303:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574597.249309:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574597.249322:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249331:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574597.249338:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249350:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574597.249357:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574597.249365:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249373:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574597.249378:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574597.249387:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574597.249398:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.249405:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574597.249410:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574597.249417:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.249422:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574597.249430:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574597.249433:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574597.249436:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574597.249438:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.249440:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574597.249442:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574597.249446:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574597.249452:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574597.249456:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574597.249465:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516717440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574597.249478:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574597.249483:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574597.249491:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516717440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574597.249505:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574597.249510:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574597.249514:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574597.249517:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574597.249523:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574597.249535:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574597.249552:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15284 00000400:00000200:0.0:1788574597.249569:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574597.249579:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574597.249585:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574597.249591:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249611:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574597.249619:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249623:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574597.249626:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574597.249630:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574597.249636:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574597.249641:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574597.249645:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574597.249651:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574597.249660:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249670:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574597.249682:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249691:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574597.249706:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574597.249718:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574597.249724:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574597.249736:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574597.249746:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574597.249750:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.249753:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574597.249758:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574597.249765:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.249768:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574597.249774:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574597.249791:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574597.249801:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574597.249896:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.249907:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574597.249914:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574597.249921:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574597.249928:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574597.249943:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574597.249955:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516717568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574597.249972:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574597.249982:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574597.249993:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516717568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574597.250011:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574597.250021:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574597.250029:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574597.250033:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574597.250037:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574597.250045:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574597.250057:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15284 00000400:00000200:0.0:1788574597.250078:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574597.250088:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574597.250093:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574597.250103:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.250109:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574597.250118:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.250134:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574597.250146:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574597.250150:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574597.250165:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574597.250168:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574597.250176:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574597.250182:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574597.250194:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.250200:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574597.250209:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.250215:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574597.250223:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574597.250234:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574597.250248:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574597.250255:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574597.250263:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574597.250268:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.250271:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574597.250278:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574597.250287:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.250295:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574597.250305:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574597.250315:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574597.250321:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574597.250325:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.250337:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574597.250345:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574597.250348:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574597.250355:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574597.250360:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574597.250380:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516717696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574597.250404:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574597.250409:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574597.250423:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516717696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574597.250483:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574597.250491:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574597.250499:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574597.250503:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574597.250512:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574597.250526:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574597.250542:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15284 00000400:00000200:0.0:1788574597.250568:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574597.250588:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574597.250617:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574597.250633:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.250641:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574597.250653:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.250662:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574597.250672:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574597.250677:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574597.250688:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574597.250698:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574597.250708:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574597.250712:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574597.250718:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.250726:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574597.250735:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574597.250746:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574597.250756:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574597.250772:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574597.250783:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574597.250806:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574597.250845:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574597.250849:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.250852:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574597.250857:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574597.250860:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.250861:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574597.250866:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574597.250870:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574597.250877:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574597.250882:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574597.250890:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574597.250896:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574597.250902:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574597.250912:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574597.250918:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574597.250929:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516717824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574597.250971:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574597.250979:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574597.250998:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516717824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574597.251036:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574597.251046:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15294) 00000100:00000001:0.0:1788574597.761022:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574597.761030:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574597.761033:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574597.761066:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574597.761082:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574597.761085:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574597.762223:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574597.762225:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574597.762253:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574598.465003:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574598.465026:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574598.465030:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574598.784915:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574598.784922:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574598.784925:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574598.784931:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574598.784937:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574598.784941:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574598.784943:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574598.784945:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574598.784948:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574598.785009:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574598.785016:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574598.785020:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574598.785033:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574598.785040:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574598.785043:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574598.785060:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574598.785063:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574598.785066:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574599.808925:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574599.808932:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574599.808938:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574599.808944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574599.808952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574599.808961:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574599.808997:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574599.809002:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574599.809008:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574600.513119:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574600.513123:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574600.513126:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574600.832955:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574600.832965:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574600.832968:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574600.832977:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574600.832983:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574600.832988:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574600.833014:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574600.833017:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574600.833020:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574601.088950:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309960942, at 4309960960 00000100:00000010:0.0:1788574601.088971:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abae0. 00000020:00000001:0.0:1788574601.537042:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574601.537045:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574601.537048:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574601.857004:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574601.857011:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574601.857015:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574601.857020:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574601.857024:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574601.857027:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574601.857030:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574601.857033:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574601.857036:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574602.368978:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574602.368991:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574602.368995:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574602.368999:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574602.369004:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574602.369011:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15289 00000400:00000200:0.0:1788574602.369021:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574602.369028:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574602.369031:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574602.369041:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369044:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574602.369046:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369049:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574602.369052:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574602.369054:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574602.369057:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574602.369086:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574602.369089:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574602.369092:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574602.369096:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369099:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574602.369102:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369105:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574602.369109:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574602.369113:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369116:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574602.369119:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574602.369122:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574602.369125:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.369127:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574602.369129:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574602.369136:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574602.369140:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574602.369144:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574602.369146:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.369149:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574602.369152:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574602.369156:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574602.369164:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574602.369171:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574602.369180:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516717952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574602.369190:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574602.369194:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574602.369198:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516717952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574602.369219:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574602.369222:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574602.369229:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574602.369231:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574602.369233:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574602.369236:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574602.369238:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15289 00000400:00000200:0.0:1788574602.369242:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574602.369245:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574602.369250:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574602.369253:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369258:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574602.369262:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369265:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574602.369268:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574602.369269:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574602.369273:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574602.369276:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574602.369281:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574602.369286:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574602.369290:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369294:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574602.369296:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369298:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574602.369299:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574602.369301:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369302:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574602.369304:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574602.369310:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574602.369313:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.369319:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574602.369325:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574602.369334:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.369337:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574602.369342:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574602.369345:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574602.369347:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574602.369348:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.369349:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574602.369350:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574602.369354:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574602.369359:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574602.369364:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574602.369372:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516718080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574602.369385:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574602.369388:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574602.369393:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516718080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574602.369403:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574602.369405:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574602.369411:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574602.369415:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574602.369417:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574602.369420:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574602.369426:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15289 00000400:00000200:0.0:1788574602.369436:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574602.369441:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574602.369446:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574602.369460:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369465:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574602.369468:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369484:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574602.369491:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574602.369494:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574602.369497:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574602.369501:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574602.369505:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574602.369511:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574602.369520:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369524:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574602.369530:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369537:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574602.369542:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574602.369547:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369551:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574602.369554:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574602.369557:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574602.369561:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.369579:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574602.369581:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574602.369583:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.369584:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574602.369588:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574602.369590:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574602.369592:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574602.369594:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.369596:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574602.369599:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574602.369602:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574602.369609:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574602.369616:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574602.369623:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516718208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574602.369655:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574602.369657:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574602.369665:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516718208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574602.369682:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574602.369687:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574602.369691:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574602.369692:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574602.369693:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574602.369698:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574602.369702:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15289 00000400:00000200:0.0:1788574602.369705:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574602.369708:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574602.369712:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574602.369718:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369721:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574602.369727:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369729:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574602.369731:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574602.369732:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574602.369733:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574602.369736:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574602.369740:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574602.369745:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574602.369747:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369749:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574602.369751:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369752:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574602.369761:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574602.369768:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369773:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574602.369778:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574602.369781:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574602.369784:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.369787:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574602.369792:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574602.369799:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.369804:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574602.369839:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574602.369843:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574602.369848:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574602.369851:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.369853:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574602.369857:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574602.369860:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574602.369865:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574602.369870:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574602.369878:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516718336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574602.369890:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574602.369895:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574602.369901:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516718336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574602.369912:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574602.369914:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574602.369922:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574602.369926:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574602.369929:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574602.369933:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574602.369943:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15289 00000400:00000200:0.0:1788574602.369952:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574602.369960:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574602.369963:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574602.369965:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369970:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574602.369974:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.369979:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574602.369982:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574602.369985:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574602.369987:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574602.369991:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574602.369996:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574602.370000:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574602.370007:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.370010:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574602.370014:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574602.370017:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574602.370021:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574602.370027:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574602.370030:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574602.370034:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574602.370038:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574602.370041:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.370047:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574602.370048:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574602.370051:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.370053:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574602.370057:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574602.370063:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574602.370070:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574602.370079:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574602.370087:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574602.370091:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574602.370097:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574602.370107:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574602.370114:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574602.370122:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516718464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574602.370131:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574602.370136:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574602.370144:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516718464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574602.370159:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574602.370165:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15299) 00000100:00000001:0.0:1788574602.880964:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574602.880970:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574602.880975:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574602.880982:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574602.880986:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574602.880990:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574602.881017:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574602.881020:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574602.881024:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574603.585118:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574603.585125:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574603.585140:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574603.905002:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574603.905009:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574603.905015:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574603.905022:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574603.905035:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574603.905047:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574603.905084:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574603.905087:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574603.905090:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574604.928929:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574604.928938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574604.928941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574604.928948:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574604.928953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574604.928957:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574604.928961:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574604.928965:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574604.928970:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574605.633034:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574605.633038:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574605.633042:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574605.952937:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574605.952957:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574605.952965:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574605.952977:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574605.952982:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574605.952989:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574605.952999:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574605.953011:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574605.953016:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574606.208905:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309966062, at 4309966080 00000100:00000010:0.0:1788574606.208924:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab900. 00000020:00000001:0.0:1788574606.657014:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574606.657017:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574606.657021:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788574607.488908:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574607.488916:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574607.488919:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574607.488922:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574607.488926:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574607.488933:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15294 00000400:00000200:0.0:1788574607.488941:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574607.488947:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574607.488949:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574607.488968:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.488970:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574607.488972:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.488974:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574607.488977:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574607.488979:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574607.488981:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574607.488983:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574607.488985:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574607.488987:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574607.488989:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.488991:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574607.488992:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.488993:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574607.488996:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574607.488998:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574607.488999:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574607.489002:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574607.489004:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574607.489006:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.489008:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574607.489010:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574607.489015:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574607.489018:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574607.489020:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574607.489021:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.489023:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574607.489025:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574607.489029:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574607.489035:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574607.489040:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574607.489047:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516718592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574607.489058:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574607.489062:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574607.489065:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516718592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574607.489090:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574607.489093:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574607.489097:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574607.489098:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574607.489101:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574607.489104:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574607.489107:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15294 00000400:00000200:0.0:1788574607.489114:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574607.489119:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574607.489122:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574607.489130:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489145:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574607.489151:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489153:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574607.489155:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574607.489156:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574607.489158:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574607.489162:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574607.489170:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574607.489179:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574607.489191:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489197:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574607.489209:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489222:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574607.489235:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574607.489244:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489251:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574607.489255:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574607.489261:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574607.489265:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.489271:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574607.489275:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574607.489278:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.489279:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574607.489283:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574607.489286:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574607.489289:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574607.489290:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.489291:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574607.489293:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574607.489297:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574607.489302:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574607.489307:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574607.489315:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516718720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574607.489329:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574607.489331:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574607.489334:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516718720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574607.489349:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574607.489367:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574607.489376:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574607.489379:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574607.489381:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574607.489386:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574607.489401:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15294 00000400:00000200:0.0:1788574607.489414:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574607.489418:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574607.489419:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574607.489422:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489426:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574607.489432:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489436:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574607.489444:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574607.489445:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574607.489447:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574607.489449:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574607.489459:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574607.489462:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574607.489469:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489473:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574607.489477:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489479:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574607.489482:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574607.489490:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489493:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574607.489498:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574607.489502:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574607.489505:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.489511:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574607.489512:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574607.489517:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.489520:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574607.489524:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574607.489530:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574607.489552:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574607.489559:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.489562:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574607.489567:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574607.489574:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574607.489581:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574607.489587:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574607.489595:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516718848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574607.489622:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574607.489631:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574607.489637:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516718848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574607.489651:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574607.489661:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574607.489676:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574607.489683:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574607.489685:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574607.489687:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574607.489693:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15294 00000400:00000200:0.0:1788574607.489706:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574607.489714:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574607.489720:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574607.489728:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489733:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574607.489736:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489740:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574607.489749:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574607.489751:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574607.489755:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574607.489764:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574607.489769:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574607.489783:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574607.489789:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489793:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574607.489798:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489806:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574607.489846:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574607.489860:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574607.489863:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574607.489868:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574607.489873:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574607.489885:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.489890:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574607.489893:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574607.489898:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.489899:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574607.489903:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574607.489905:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574607.489908:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574607.489910:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.489912:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574607.489914:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574607.489916:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574607.489920:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574607.489925:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574607.489933:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516718976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574607.489948:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574607.489950:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574607.489957:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516718976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574607.489965:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574607.489974:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574607.489978:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574607.489982:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574607.489983:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574607.489985:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574607.489987:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15294 00000400:00000200:0.0:1788574607.489998:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574607.490011:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574607.490014:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574607.490018:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.490023:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574607.490031:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.490038:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574607.490040:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574607.490044:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574607.490050:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574607.490056:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574607.490065:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574607.490071:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574607.490081:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.490086:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574607.490092:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574607.490098:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574607.490105:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574607.490117:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574607.490127:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574607.490138:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574607.490147:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574607.490154:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.490165:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574607.490177:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574607.490184:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.490190:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574607.490198:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574607.490207:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574607.490217:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574607.490222:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574607.490227:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574607.490231:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574607.490235:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574607.490241:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574607.490247:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574607.490260:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516719104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574607.490278:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574607.490282:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574607.490292:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516719104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574607.490308:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574607.490314:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15305) 00000100:00000001:0.0:1788574608.000972:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574608.000989:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574608.000998:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574608.001014:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574608.001023:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574608.001031:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574608.001038:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574608.001049:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574608.001062:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574608.705013:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574608.705016:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574608.705018:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574609.025004:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574609.025010:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574609.025039:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574609.025047:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574609.025054:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574609.025059:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574609.025063:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574609.025066:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574609.025069:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574609.537213:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788574610.048906:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574610.048913:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574610.048918:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574610.048925:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574610.048931:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574610.048936:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574610.048943:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574610.048947:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574610.048953:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574610.752983:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574610.752986:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574610.752989:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574611.072918:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574611.072922:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574611.072925:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574611.072930:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574611.072934:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574611.072938:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574611.072942:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574611.072946:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574611.072954:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574611.072971:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574611.072975:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574611.072979:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574611.072982:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574611.072988:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574611.072992:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574611.072996:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574611.072997:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574611.072999:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574611.328912:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309971182, at 4309971200 00000100:00000010:0.0:1788574611.328927:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab0c0. 00000020:00000001:0.0:1788574611.777010:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574611.777014:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574611.777016:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574612.096940:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574612.096946:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574612.096950:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574612.096956:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574612.096963:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574612.096968:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574612.096996:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574612.097000:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574612.097004:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574612.608893:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574612.608902:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574612.608905:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574612.608909:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574612.608913:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574612.608920:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15300 00000400:00000200:0.0:1788574612.608929:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574612.608935:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574612.608938:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574612.608942:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.608945:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574612.608947:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.608949:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574612.608953:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574612.608954:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574612.608956:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574612.608958:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574612.608960:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574612.608962:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574612.608980:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.608982:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574612.608983:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.608985:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574612.608987:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574612.608989:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574612.608991:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574612.608993:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574612.608996:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574612.608998:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.608999:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574612.609001:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574612.609005:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574612.609008:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574612.609011:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574612.609013:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609015:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574612.609017:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574612.609020:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574612.609026:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574612.609032:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574612.609038:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516719232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574612.609046:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574612.609049:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574612.609053:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516719232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574612.609069:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574612.609071:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574612.609074:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574612.609075:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574612.609079:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574612.609082:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574612.609084:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15300 00000400:00000200:0.0:1788574612.609088:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574612.609091:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574612.609093:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574612.609095:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609097:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574612.609100:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609102:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574612.609105:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574612.609106:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574612.609111:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574612.609114:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574612.609119:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574612.609123:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574612.609144:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609149:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574612.609155:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609160:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574612.609166:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574612.609173:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609176:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574612.609179:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574612.609185:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574612.609189:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609191:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574612.609195:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574612.609198:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609204:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574612.609213:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574612.609217:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574612.609220:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574612.609223:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609227:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574612.609229:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574612.609231:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574612.609235:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574612.609240:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574612.609244:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516719360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574612.609254:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574612.609258:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574612.609266:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516719360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574612.609289:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574612.609293:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574612.609297:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574612.609300:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574612.609303:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574612.609307:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574612.609310:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15300 00000400:00000200:0.0:1788574612.609317:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574612.609323:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574612.609325:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574612.609331:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609334:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574612.609338:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609340:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574612.609342:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574612.609347:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574612.609350:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574612.609352:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574612.609354:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574612.609358:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574612.609361:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609362:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574612.609363:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609366:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574612.609369:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574612.609373:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609376:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574612.609380:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574612.609382:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574612.609386:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609390:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574612.609392:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574612.609396:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609400:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574612.609405:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574612.609407:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574612.609412:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574612.609414:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609418:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574612.609420:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574612.609423:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574612.609427:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574612.609432:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574612.609436:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516719488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574612.609449:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574612.609453:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574612.609464:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516719488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574612.609474:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574612.609477:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574612.609483:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574612.609486:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574612.609489:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574612.609492:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574612.609495:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15300 00000400:00000200:0.0:1788574612.609525:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574612.609532:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574612.609537:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574612.609542:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609544:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574612.609548:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609551:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574612.609553:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574612.609554:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574612.609556:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574612.609558:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574612.609561:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574612.609563:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574612.609570:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609574:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574612.609578:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609580:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574612.609582:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574612.609588:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609594:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574612.609600:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574612.609604:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574612.609607:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609609:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574612.609612:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574612.609617:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609621:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574612.609626:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574612.609629:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574612.609631:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574612.609634:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609636:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574612.609639:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574612.609642:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574612.609648:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574612.609652:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574612.609660:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516719616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574612.609672:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574612.609675:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574612.609681:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516719616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574612.609691:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574612.609694:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574612.609697:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574612.609697:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574612.609699:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574612.609701:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574612.609703:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15300 00000400:00000200:0.0:1788574612.609706:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574612.609708:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574612.609709:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574612.609727:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609729:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574612.609733:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609736:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574612.609741:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574612.609743:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574612.609746:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574612.609749:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574612.609754:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574612.609760:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574612.609764:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609766:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574612.609770:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609775:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574612.609779:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574612.609784:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574612.609788:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574612.609790:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574612.609795:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574612.609801:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609804:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574612.609833:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574612.609838:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609841:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574612.609845:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574612.609850:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574612.609854:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574612.609856:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574612.609863:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574612.609864:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574612.609868:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574612.609871:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574612.609882:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574612.609892:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516719744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574612.609902:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574612.609905:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574612.609910:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516719744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574612.609937:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574612.609942:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15310) 00000100:00000001:0.0:1788574613.121031:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574613.121036:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574613.121038:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574613.121041:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574613.121044:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574613.121046:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574613.121048:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574613.121050:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574613.121052:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574613.824970:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574613.824974:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574613.824977:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574614.144971:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574614.144981:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574614.144985:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574614.144994:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574614.145000:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574614.145003:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574614.145006:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574614.145012:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574614.145020:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574615.168922:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574615.168930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574615.168934:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574615.168941:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574615.168946:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574615.168957:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574615.168975:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574615.168978:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574615.168981:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574615.873023:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574615.873028:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574615.873031:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574616.192958:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574616.192966:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574616.192971:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574616.192977:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574616.192984:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574616.192990:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574616.192995:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574616.192998:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574616.193001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574616.448888:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309976301, at 4309976320 00000100:00000010:0.0:1788574616.448901:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab240. 00000020:00000001:0.0:1788574616.896993:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574616.896998:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574616.897002:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574617.216923:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574617.216931:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574617.216935:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574617.216943:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574617.216949:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574617.216956:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574617.216963:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574617.216967:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574617.216971:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.217042:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574617.217047:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574617.217049:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574617.217053:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574617.217057:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574617.217060:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574617.217064:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574617.217079:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574617.217082:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574617.728986:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574617.729006:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574617.729011:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574617.729017:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574617.729023:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574617.729032:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15305 00000400:00000200:0.0:1788574617.729042:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574617.729048:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574617.729051:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574617.729055:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729058:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574617.729063:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729065:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574617.729069:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574617.729071:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574617.729073:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574617.729076:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574617.729077:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574617.729079:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574617.729081:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729083:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574617.729084:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729085:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574617.729087:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574617.729089:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729091:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574617.729093:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574617.729096:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574617.729098:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.729115:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574617.729117:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574617.729122:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574617.729125:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574617.729129:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574617.729130:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.729133:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574617.729135:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574617.729139:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574617.729146:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574617.729151:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574617.729159:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516719872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574617.729167:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574617.729170:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574617.729174:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516719872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574617.729193:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574617.729196:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574617.729198:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574617.729199:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574617.729201:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574617.729202:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574617.729204:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15305 00000400:00000200:0.0:1788574617.729208:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574617.729211:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574617.729213:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574617.729216:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729218:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574617.729220:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729222:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574617.729228:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574617.729231:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574617.729233:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574617.729235:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574617.729240:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574617.729242:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574617.729246:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729248:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574617.729252:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729253:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574617.729255:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574617.729260:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729264:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574617.729267:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574617.729272:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574617.729275:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.729279:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574617.729283:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574617.729286:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.729288:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574617.729292:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574617.729295:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574617.729299:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574617.729303:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.729306:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574617.729308:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574617.729312:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574617.729317:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574617.729321:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574617.729329:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516720000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574617.729339:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574617.729347:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574617.729353:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516720000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574617.729361:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574617.729365:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574617.729367:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574617.729368:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574617.729370:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574617.729372:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574617.729375:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15305 00000400:00000200:0.0:1788574617.729382:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574617.729387:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574617.729390:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574617.729396:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729401:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574617.729405:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729408:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574617.729421:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574617.729424:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574617.729428:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574617.729431:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574617.729432:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574617.729438:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574617.729443:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729447:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574617.729453:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729455:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574617.729457:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574617.729464:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729484:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574617.729489:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574617.729493:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574617.729499:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.729505:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574617.729507:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574617.729511:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.729514:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574617.729517:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574617.729522:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574617.729525:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574617.729529:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.729531:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574617.729533:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574617.729537:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574617.729542:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574617.729547:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574617.729555:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516720128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574617.729569:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574617.729573:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574617.729580:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516720128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574617.729590:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574617.729594:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574617.729598:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574617.729601:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574617.729603:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574617.729609:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574617.729614:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15305 00000400:00000200:0.0:1788574617.729621:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574617.729634:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574617.729637:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574617.729642:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729647:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574617.729651:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729653:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574617.729655:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574617.729658:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574617.729661:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574617.729663:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574617.729667:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574617.729673:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574617.729682:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729687:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574617.729692:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729698:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574617.729706:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574617.729715:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729722:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574617.729726:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574617.729730:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574617.729735:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.729739:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574617.729743:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574617.729748:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.729752:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574617.729757:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574617.729761:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574617.729766:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574617.729769:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.729771:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574617.729780:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574617.729784:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574617.729790:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574617.729796:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574617.729805:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516720256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574617.729840:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574617.729844:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574617.729853:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516720256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574617.729864:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574617.729868:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574617.729873:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574617.729876:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574617.729879:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574617.729884:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574617.729888:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15305 00000400:00000200:0.0:1788574617.729895:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574617.729899:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574617.729901:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574617.729907:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729910:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574617.729915:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729919:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574617.729922:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574617.729925:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574617.729928:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574617.729931:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574617.729936:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574617.729939:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574617.729952:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729959:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574617.729966:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729973:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574617.729978:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574617.729986:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574617.729993:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574617.729999:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574617.730005:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574617.730011:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.730016:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574617.730019:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574617.730021:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.730023:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574617.730027:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574617.730033:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574617.730039:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574617.730041:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574617.730046:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574617.730049:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574617.730053:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574617.730058:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574617.730065:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574617.730073:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516720384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574617.730085:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574617.730088:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574617.730097:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516720384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574617.730113:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574617.730119:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15315) 00000100:00000001:0.0:1788574618.240995:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574618.241011:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574618.241014:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574618.241020:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574618.241024:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574618.241029:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574618.241032:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574618.241035:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574618.241039:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574618.944990:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574618.944993:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574618.944996:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574620.288937:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574620.288941:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574620.288945:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574620.288952:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574620.288959:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574620.288964:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574620.288978:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574620.288985:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574620.288991:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574620.993023:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574620.993026:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574620.993028:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574621.312919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574621.312927:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574621.312932:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574621.312941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574621.312948:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574621.312952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574621.312965:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574621.312970:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574621.312976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574621.568910:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309981421, at 4309981440 00000100:00000010:0.0:1788574621.568927:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab360. 00000020:00000001:0.0:1788574622.016997:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574622.017001:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574622.017021:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574622.336914:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574622.336922:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574622.336926:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574622.336937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574622.336946:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574622.336953:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.336961:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574622.336966:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.336972:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.849195:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574622.849201:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574622.849204:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574622.849212:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516720512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.849222:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574622.849225:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574622.849232:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574622.849237:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574622.849242:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574622.849248:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516720512:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574622.849254:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574622.849258:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574622.849261:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574622.849266:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574622.849271:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574622.849275:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574622.849285:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574622.849290:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574622.849299:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574622.849307:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516720512, portal 25 00000100:00000001:0.0:1788574622.849312:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574622.849315:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788574622.849322:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516720512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.849348:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574622.849353:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574622.849358:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574622.849363:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516720512, offset 0 00000400:00000200:0.0:1788574622.849375:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574622.849388:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574622.849398:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574622.849405:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574622.849413:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574622.849418:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574622.849421:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574622.849453:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547281:547281:256:4294967295] 192.168.202.128@tcp LPNI seq info [547204:547204:8:4294967295] 00000400:00000200:0.0:1788574622.849461:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574622.849475:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574622.849486:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574622.849493:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574622.849504:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574622.849517:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574622.849540:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574622.849545:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574622.849553:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574622.849562:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574622.849568:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574622.849572:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574622.849579:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574622.849588:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516720512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.849601:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574622.849606:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516720512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.849636:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.849640:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574622.849648:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574622.849655:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574622.849658:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309986541 00000100:00000010:0.0:1788574622.849663:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574622.849670:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574622.849678:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309986541 00000100:00000010:0.0:1788574622.849684:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50480. 00000100:00000001:0.0:1788574622.849691:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574622.849698:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574622.849703:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.849708:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574622.849718:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574622/real 1788574622] req@ffff8fc4ac820700 x1875440516720512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574622.849736:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574622.849741:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574622.849745:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574622.849754:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574622.849758:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574622.849764:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574622.849772:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516720512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574622.849792:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516720512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.849830:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574622.849832:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.849835:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574622.849838:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574622.849845:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516720512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.849886:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574622.849897:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516720512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.849913:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574622.849917:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.849919:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574622.849921:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.849925:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574622.849930:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574622.849936:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574622.849944:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574622.849950:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574622.849959:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516720512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574622.849983:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516720512:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574622.849993:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.849997:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574622.850005:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516720512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.850023:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574622.850029:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574622.850034:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574622.850040:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574622.850044:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574622.850049:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574622.850056:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574622.850060:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788574622.850066:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574622.850075:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574622.850080:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788574622.850087:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574622.850090:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.850096:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.850105:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574622.850110:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.850118:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574622.850122:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574622.850125:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574622.850134:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516720640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.850149:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574622.850152:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574622.850156:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574622.850160:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574622.850165:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574622.850173:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516720640:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574622.850180:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574622.850184:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574622.850187:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574622.850191:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574622.850197:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574622.850201:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574622.850209:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574622.850213:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574622.850219:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574622.850226:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516720640, portal 10 00000100:00000001:0.0:1788574622.850231:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574622.850235:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788574622.850247:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516720640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.850265:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574622.850275:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574622.850286:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574622.850292:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516720640, offset 0 00000400:00000200:0.0:1788574622.850303:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574622.850311:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574622.850316:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574622.850323:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574622.850329:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574622.850337:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574622.850343:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574622.850353:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547282:547282:256:4294967295] 192.168.202.128@tcp LPNI seq info [547205:547205:8:4294967295] 00000400:00000200:0.0:1788574622.850365:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574622.850384:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574622.850396:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574622.850403:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574622.850411:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574622.850417:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574622.850455:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574622.850459:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574622.850466:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574622.850474:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574622.850482:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574622.850488:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574622.850494:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574622.850502:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516720640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.850519:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574622.850526:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516720640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.850543:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.850546:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574622.850560:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574622.850566:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574622.850571:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574622.850574:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309982739 00000100:00000001:0.0:1788574622.850578:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309986541 00000100:00000001:0.0:1788574622.850580:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309982739 00000100:00000001:0.0:1788574622.850583:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309986541 00000100:00000001:0.0:1788574622.850586:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574622.850588:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.850591:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574622.850596:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574622/real 1788574622] req@ffff8fc4ac822300 x1875440516720640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574622.850606:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574622.850607:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574622.850609:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574622.850612:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574622.850614:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574622.850617:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574622.850621:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516720640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574622.850632:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516720640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.850640:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574622.850641:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.850643:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574622.850645:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574622.850649:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516720640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.850657:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574622.850662:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516720640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.850675:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574622.850678:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.850680:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574622.850682:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.850684:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574622.850686:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574622.850689:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574622.850693:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574622.850695:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574622.850700:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516720640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574622.850710:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516720640:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574622.850714:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574622.850718:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516720768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.850725:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574622.850727:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574622.850729:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574622.850731:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574622.850733:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574622.850737:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516720768:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574622.850740:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574622.850742:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574622.850744:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574622.850746:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574622.850749:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574622.850751:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574622.850759:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574622.850762:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574622.850765:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574622.850769:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516720768, portal 10 00000100:00000001:0.0:1788574622.850771:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574622.850772:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788574622.850779:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516720768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.850786:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574622.850788:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574622.850790:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574622.850793:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516720768, offset 0 00000400:00000200:0.0:1788574622.850797:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574622.850801:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574622.850805:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574622.850828:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574622.850831:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574622.850833:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574622.850836:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574622.850839:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547283:547283:256:4294967295] 192.168.202.128@tcp LPNI seq info [547206:547206:8:4294967295] 00000400:00000200:0.0:1788574622.850845:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574622.850852:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574622.850857:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574622.850861:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574622.850865:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574622.850868:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574622.850880:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574622.850882:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574622.850885:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574622.850889:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574622.850893:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574622.850899:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574622.850903:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574622.850907:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516720768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.850916:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574622.850919:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516720768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.850926:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.850928:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574622.850931:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574622.850934:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574622.850938:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574622.850940:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309982747 00000100:00000001:0.0:1788574622.850943:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309986541 00000100:00000001:0.0:1788574622.850945:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309982747 00000100:00000001:0.0:1788574622.850947:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309986541 00000100:00000001:0.0:1788574622.850949:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574622.850951:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.850954:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574622.850959:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574622/real 1788574622] req@ffff8fc4ac823480 x1875440516720768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574622.850967:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574622.850968:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574622.850970:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574622.850973:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574622.850974:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574622.850977:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574622.850980:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516720768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574622.850987:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516720768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851003:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574622.851004:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851006:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574622.851007:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574622.851012:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516720768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851019:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574622.851023:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516720768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851030:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574622.851033:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851034:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574622.851036:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851038:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574622.851041:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574622.851044:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574622.851048:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574622.851050:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574622.851054:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516720768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574622.851063:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516720768:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574622.851067:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851069:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574622.851073:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516720640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851080:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574622.851087:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574622.851090:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574622.851093:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574622.851094:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574622.851096:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574622.851098:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574622.851100:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788574622.851102:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574622.851104:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574622.851106:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788574622.851109:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574622.851110:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851112:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574622.851116:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516720768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851123:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574622.851125:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574622.851126:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574622.851129:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574622.851130:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574622.851132:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574622.851134:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574622.851135:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788574622.851139:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574622.851140:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574622.851142:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788574622.851144:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574622.851146:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851148:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851152:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574622.851154:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.851156:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574622.851158:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574622.851159:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574622.851163:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516720896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851170:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574622.851176:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574622.851179:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574622.851181:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574622.851183:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574622.851186:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516720896:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574622.851190:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574622.851192:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574622.851194:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574622.851195:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574622.851198:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574622.851200:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574622.851205:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574622.851208:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574622.851212:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574622.851215:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516720896, portal 4 00000100:00000001:0.0:1788574622.851217:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574622.851218:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788574622.851225:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516720896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851232:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574622.851234:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574622.851237:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574622.851239:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516720896, offset 0 00000400:00000200:0.0:1788574622.851243:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574622.851247:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574622.851251:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574622.851255:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574622.851257:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574622.851260:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574622.851263:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574622.851266:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547284:547284:256:4294967295] 192.168.202.128@tcp LPNI seq info [547207:547207:8:4294967295] 00000400:00000200:0.0:1788574622.851271:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574622.851278:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574622.851300:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574622.851304:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574622.851308:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574622.851311:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574622.851323:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574622.851325:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574622.851328:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574622.851332:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574622.851335:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574622.851337:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574622.851340:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574622.851344:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516720896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851353:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574622.851356:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516720896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851363:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.851364:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574622.851368:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574622.851372:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574622.851375:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574622.851378:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309982887 00000100:00000001:0.0:1788574622.851380:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309986541 00000100:00000001:0.0:1788574622.851383:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309982887 00000100:00000001:0.0:1788574622.851385:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309986541 00000100:00000001:0.0:1788574622.851387:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574622.851389:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.851391:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574622.851395:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574622/real 1788574622] req@ffff8fc4ac821f80 x1875440516720896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574622.851409:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574622.851411:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574622.851413:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574622.851415:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574622.851417:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574622.851419:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574622.851424:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516720896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574622.851448:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516720896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851455:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574622.851456:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851459:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574622.851460:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574622.851464:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516720896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851470:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574622.851475:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516720896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851482:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574622.851485:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851486:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574622.851488:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851490:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574622.851492:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574622.851495:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574622.851499:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574622.851502:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574622.851514:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516720896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574622.851524:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516720896:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574622.851528:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574622.851531:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516721024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851538:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574622.851540:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574622.851543:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574622.851545:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574622.851548:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574622.851551:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516721024:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574622.851554:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574622.851557:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574622.851558:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574622.851560:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574622.851564:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574622.851566:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574622.851570:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574622.851573:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574622.851577:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574622.851580:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516721024, portal 4 00000100:00000001:0.0:1788574622.851583:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574622.851584:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788574622.851591:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516721024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851598:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574622.851600:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574622.851603:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574622.851605:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516721024, offset 0 00000400:00000200:0.0:1788574622.851617:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574622.851622:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574622.851626:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574622.851629:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574622.851632:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574622.851634:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574622.851637:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574622.851640:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547285:547285:256:4294967295] 192.168.202.128@tcp LPNI seq info [547208:547208:8:4294967295] 00000400:00000200:0.0:1788574622.851644:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574622.851652:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574622.851656:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574622.851661:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574622.851665:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574622.851668:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574622.851679:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574622.851681:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574622.851684:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574622.851688:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574622.851691:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574622.851693:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574622.851696:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574622.851700:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516721024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851709:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574622.851712:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516721024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574647 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851719:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.851721:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574622.851724:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574622.851728:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574622.851731:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574622.851738:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309982889 00000100:00000001:0.0:1788574622.851740:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309986541 00000100:00000001:0.0:1788574622.851742:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309982889 00000100:00000001:0.0:1788574622.851743:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309986541 00000100:00000001:0.0:1788574622.851744:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574622.851746:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.851747:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574622.851751:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574622/real 1788574622] req@ffff8fc4ac822d80 x1875440516721024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574622.851759:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574622.851760:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574622.851762:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574622.851765:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574622.851767:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574622.851769:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574622.851773:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516721024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574622.851781:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516721024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851789:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574622.851790:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851792:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574622.851794:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574622.851797:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516721024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851804:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574622.851828:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516721024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851834:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574622.851841:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851842:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574622.851843:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851845:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574622.851846:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574622.851848:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574622.851851:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574622.851852:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574622.851855:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516721024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574622.851860:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516721024:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574622.851863:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851864:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574622.851866:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516720896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851870:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574622.851872:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574622.851874:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574622.851877:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574622.851879:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574622.851880:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574622.851882:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574622.851883:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574622.851886:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574622.851887:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574622.851888:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788574622.851890:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574622.851891:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851892:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574622.851895:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516721024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574647 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574622.851906:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574622.851908:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574622.851909:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574622.851910:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574622.851911:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574622.851912:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574622.851913:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574622.851914:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788574622.851915:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574622.851916:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574622.851916:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788574622.851918:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574622.851918:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851920:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574622.851922:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574622.851923:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574622.851924:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574622.851925:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574623.360973:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574623.360979:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574623.360981:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574623.360985:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574623.360993:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574623.360998:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574623.361013:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574623.361015:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574623.361018:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574624.064999:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574624.065003:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574624.065006:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574624.384978:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574624.384989:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574624.384994:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574624.385002:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574624.385011:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574624.385016:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574624.385044:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574624.385047:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574624.385053:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574625.408968:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574625.408986:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574625.408990:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574625.408997:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574625.409004:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574625.409011:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574625.409018:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574625.409028:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574625.409044:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574626.113001:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574626.113004:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574626.113007:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574626.432912:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574626.432920:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574626.432923:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574626.432928:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574626.432933:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574626.432937:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574626.432943:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574626.432947:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574626.432951:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574626.432994:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574626.433000:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574626.433003:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574626.433014:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574626.433018:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574626.433026:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574626.433032:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574626.433034:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574626.433037:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574626.688922:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309986541, at 4309986560 00000100:00000010:0.0:1788574626.688939:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574627.137016:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574627.137021:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574627.137041:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574627.456937:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574627.456942:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574627.456946:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574627.456950:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574627.456955:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574627.456960:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.456972:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574627.456976:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.456980:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.969201:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574627.969210:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574627.969215:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574627.969234:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516721152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.969299:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574627.969307:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574627.969311:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574627.969315:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574627.969321:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574627.969332:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516721152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574627.969340:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574627.969345:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574627.969349:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574627.969356:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574627.969368:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574627.969389:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574627.969408:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574627.969413:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574627.969421:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574627.969435:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516721152, portal 25 00000100:00000001:0.0:1788574627.969439:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574627.969442:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788574627.969470:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516721152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.969541:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574627.969546:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574627.969557:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574627.969565:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516721152, offset 0 00000400:00000200:0.0:1788574627.969587:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574627.969600:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574627.969613:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574627.969626:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574627.969635:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574627.969642:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574627.969649:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574627.969663:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547286:547286:256:4294967295] 192.168.202.128@tcp LPNI seq info [547209:547209:8:4294967295] 00000400:00000200:0.0:1788574627.969677:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574627.969698:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574627.969705:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574627.969715:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574627.969724:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574627.969733:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574627.969779:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574627.969783:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574627.969793:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574627.969804:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574627.969834:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574627.969839:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574627.969851:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574627.969864:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516721152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.969897:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574627.969911:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516721152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.969960:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.969967:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574627.969975:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574627.969982:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574627.969988:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309991662 00000100:00000010:0.0:1788574627.969996:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574627.970006:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574627.970017:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309991662 00000100:00000010:0.0:1788574627.970026:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50ea0. 00000100:00000001:0.0:1788574627.970032:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574627.970044:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574627.970051:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.970062:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574627.970077:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574627/real 1788574627] req@ffff8fc4ac822a00 x1875440516721152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574627.970108:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574627.970114:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574627.970124:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574627.970136:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574627.970140:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574627.970146:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574627.970154:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516721152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574627.970168:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516721152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970184:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574627.970189:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.970193:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574627.970196:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574627.970206:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516721152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970257:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574627.970275:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516721152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970306:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574627.970312:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.970316:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574627.970318:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.970325:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574627.970330:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574627.970341:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574627.970354:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574627.970358:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574627.970367:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516721152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574627.970393:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516721152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574627.970404:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.970408:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574627.970413:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516721152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970432:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574627.970443:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574627.970447:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574627.970461:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574627.970469:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574627.970476:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574627.970484:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574627.970494:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788574627.970503:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574627.970514:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574627.970522:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788574627.970533:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574627.970536:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.970541:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.970549:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574627.970552:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.970560:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574627.970563:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574627.970569:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574627.970579:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516721280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970589:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574627.970591:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574627.970594:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574627.970598:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574627.970604:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574627.970609:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516721280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574627.970613:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574627.970618:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574627.970621:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574627.970631:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574627.970637:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574627.970648:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574627.970667:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574627.970678:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574627.970689:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574627.970695:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516721280, portal 10 00000100:00000001:0.0:1788574627.970698:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574627.970700:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788574627.970710:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516721280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970716:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574627.970718:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574627.970725:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574627.970730:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516721280, offset 0 00000400:00000200:0.0:1788574627.970734:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574627.970737:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574627.970741:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574627.970743:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574627.970745:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574627.970747:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574627.970748:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574627.970751:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547287:547287:256:4294967295] 192.168.202.128@tcp LPNI seq info [547210:547210:8:4294967295] 00000400:00000200:0.0:1788574627.970754:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574627.970758:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574627.970761:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574627.970764:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574627.970766:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574627.970768:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574627.970777:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574627.970779:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574627.970781:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574627.970783:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574627.970786:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574627.970787:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574627.970789:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574627.970792:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516721280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970797:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574627.970799:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516721280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970804:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.970805:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574627.970836:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574627.970840:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574627.970844:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574627.970847:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309987860 00000100:00000001:0.0:1788574627.970849:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309991662 00000100:00000001:0.0:1788574627.970852:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309987860 00000100:00000001:0.0:1788574627.970854:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309991662 00000100:00000001:0.0:1788574627.970856:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574627.970859:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.970861:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574627.970866:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574627/real 1788574627] req@ffff8fc4ac823100 x1875440516721280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574627.970875:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574627.970876:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574627.970878:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574627.970881:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574627.970882:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574627.970884:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574627.970887:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516721280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574627.970892:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516721280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970896:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574627.970897:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.970899:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574627.970900:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574627.970902:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516721280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970906:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574627.970909:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516721280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970918:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574627.970920:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.970921:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574627.970922:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.970923:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574627.970925:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574627.970927:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574627.970929:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574627.970931:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574627.970934:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516721280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574627.970939:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516721280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574627.970942:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574627.970944:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516721408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970948:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574627.970949:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574627.970951:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574627.970952:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574627.970954:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574627.970956:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516721408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574627.970958:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574627.970960:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574627.970961:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574627.970962:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574627.970965:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574627.970966:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574627.970973:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574627.970974:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574627.970976:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574627.970979:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516721408, portal 10 00000100:00000001:0.0:1788574627.970980:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574627.970981:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788574627.970986:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516721408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.970990:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574627.970991:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574627.970993:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574627.970994:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516721408, offset 0 00000400:00000200:0.0:1788574627.970997:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574627.971000:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574627.971002:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574627.971004:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574627.971006:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574627.971008:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574627.971009:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574627.971012:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547288:547288:256:4294967295] 192.168.202.128@tcp LPNI seq info [547211:547211:8:4294967295] 00000400:00000200:0.0:1788574627.971015:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574627.971019:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574627.971022:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574627.971024:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574627.971027:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574627.971028:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574627.971036:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574627.971037:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574627.971039:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574627.971041:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574627.971043:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574627.971048:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574627.971050:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574627.971053:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516721408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971058:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574627.971060:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516721408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971064:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.971065:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574627.971067:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574627.971070:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574627.971072:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574627.971073:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309987867 00000100:00000001:0.0:1788574627.971075:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309991662 00000100:00000001:0.0:1788574627.971076:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309987867 00000100:00000001:0.0:1788574627.971078:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309991662 00000100:00000001:0.0:1788574627.971079:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574627.971080:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.971081:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574627.971084:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574627/real 1788574627] req@ffff8fc4ac821500 x1875440516721408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574627.971088:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574627.971089:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574627.971090:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574627.971092:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574627.971093:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574627.971095:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574627.971097:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516721408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574627.971102:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516721408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971109:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574627.971110:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971111:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574627.971112:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574627.971115:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516721408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971119:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574627.971121:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516721408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971126:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574627.971128:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971129:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574627.971129:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971130:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574627.971132:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574627.971134:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574627.971136:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574627.971137:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574627.971139:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516721408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574627.971144:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516721408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574627.971147:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574627.971149:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516721536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971154:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574627.971155:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574627.971159:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574627.971160:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574627.971162:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574627.971164:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516721536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574627.971166:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574627.971168:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574627.971170:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574627.971171:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574627.971173:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574627.971174:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574627.971176:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574627.971178:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574627.971180:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574627.971182:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516721536, portal 4 00000100:00000001:0.0:1788574627.971184:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574627.971185:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788574627.971189:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516721536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971193:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574627.971194:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574627.971196:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574627.971197:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516721536, offset 0 00000400:00000200:0.0:1788574627.971200:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574627.971202:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574627.971204:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574627.971206:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574627.971208:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574627.971209:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574627.971211:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574627.971213:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547289:547289:256:4294967295] 192.168.202.128@tcp LPNI seq info [547212:547212:8:4294967295] 00000400:00000200:0.0:1788574627.971215:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574627.971220:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574627.971229:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574627.971232:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574627.971236:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574627.971239:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574627.971250:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574627.971252:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574627.971255:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574627.971259:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574627.971262:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574627.971264:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574627.971267:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574627.971271:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516721536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971278:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574627.971281:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516721536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971285:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.971286:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574627.971288:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574627.971291:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574627.971293:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574627.971295:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309988007 00000100:00000001:0.0:1788574627.971297:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309991662 00000100:00000001:0.0:1788574627.971298:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309988007 00000100:00000001:0.0:1788574627.971299:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309991662 00000100:00000001:0.0:1788574627.971301:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574627.971302:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.971303:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574627.971305:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574627/real 1788574627] req@ffff8fc4ac823b80 x1875440516721536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574627.971316:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574627.971316:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574627.971318:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574627.971319:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574627.971321:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574627.971322:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574627.971324:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516721536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574627.971330:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516721536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971334:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574627.971335:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971336:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574627.971337:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574627.971339:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516721536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971343:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574627.971345:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516721536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971349:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574627.971351:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971352:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574627.971353:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971354:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574627.971356:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574627.971357:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574627.971360:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574627.971361:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574627.971366:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516721536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574627.971372:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516721536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574627.971385:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971387:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574627.971390:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516721280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971394:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574627.971395:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574627.971397:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574627.971399:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574627.971400:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574627.971401:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574627.971403:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574627.971404:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788574627.971406:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574627.971407:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574627.971408:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788574627.971409:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574627.971410:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971411:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574627.971413:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516721408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971418:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574627.971418:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574627.971419:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574627.971420:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574627.971421:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574627.971422:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574627.971423:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574627.971424:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788574627.971426:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574627.971430:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574627.971432:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788574627.971433:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574627.971434:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971435:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574627.971438:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516721536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971442:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574627.971443:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574627.971444:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574627.971446:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574627.971446:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574627.971447:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574627.971448:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574627.971449:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788574627.971450:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574627.971451:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574627.971452:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788574627.971453:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574627.971454:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971455:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971458:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574627.971459:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.971460:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574627.971461:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574627.971462:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574627.971465:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516721664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971470:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574627.971471:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574627.971473:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574627.971474:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574627.971475:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574627.971477:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516721664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574627.971479:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574627.971483:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574627.971484:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574627.971485:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574627.971487:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574627.971489:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574627.971491:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574627.971493:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574627.971495:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574627.971497:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516721664, portal 4 00000100:00000001:0.0:1788574627.971499:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574627.971500:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788574627.971504:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516721664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971508:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574627.971510:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574627.971511:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574627.971513:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516721664, offset 0 00000400:00000200:0.0:1788574627.971515:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574627.971517:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574627.971520:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574627.971522:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574627.971523:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574627.971525:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574627.971527:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574627.971529:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547290:547290:256:4294967295] 192.168.202.128@tcp LPNI seq info [547213:547213:8:4294967295] 00000400:00000200:0.0:1788574627.971532:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574627.971536:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574627.971539:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574627.971541:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574627.971544:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574627.971545:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574627.971553:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574627.971557:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574627.971559:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574627.971561:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574627.971564:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574627.971565:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574627.971566:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574627.971569:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516721664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971574:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574627.971576:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516721664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574652 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971580:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.971581:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574627.971583:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574627.971585:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574627.971588:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574627.971589:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309988009 00000100:00000001:0.0:1788574627.971591:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309991662 00000100:00000001:0.0:1788574627.971592:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309988009 00000100:00000001:0.0:1788574627.971593:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309991662 00000100:00000001:0.0:1788574627.971595:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574627.971596:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.971597:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574627.971599:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574627/real 1788574627] req@ffff8fc4ac820000 x1875440516721664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574627.971604:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574627.971605:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574627.971606:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915528. 00000400:00000200:0.0:1788574627.971607:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9ee0 00000400:00000010:0.0:1788574627.971608:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9ee0. 00000100:00000001:0.0:1788574627.971613:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574627.971615:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516721664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574627.971621:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516721664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971625:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574627.971626:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971627:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574627.971628:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574627.971630:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516721664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971634:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574627.971637:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516721664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971641:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574627.971642:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971643:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574627.971644:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971645:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574627.971646:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574627.971648:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574627.971650:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574627.971651:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574627.971653:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516721664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574627.971658:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516721664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574627.971660:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971662:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574627.971666:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516721664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574652 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574627.971671:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574627.971672:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574627.971673:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574627.971675:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574627.971675:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574627.971676:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574627.971678:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574627.971679:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788574627.971681:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574627.971681:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574627.971683:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788574627.971684:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574627.971685:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971686:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574627.971689:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574627.971690:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574627.971691:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574627.971692:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574628.480920:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574628.480926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574628.480928:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574628.480933:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574628.480938:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574628.480941:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574628.480946:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574628.480950:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574628.480957:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574628.480985:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574628.480991:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574628.480995:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574628.481000:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574628.481003:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574628.481006:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574628.481009:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574628.481013:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574628.481028:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574629.185042:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574629.185047:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574629.185050:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574629.504943:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574629.504950:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574629.504953:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574629.504961:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574629.504970:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574629.504976:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574629.504993:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574629.505002:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574629.505007:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574630.528944:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574630.528954:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574630.528959:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574630.528967:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574630.528975:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574630.528981:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574630.528992:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574630.528997:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574630.529002:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574631.232996:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574631.233001:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574631.233005:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574631.552921:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574631.552929:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574631.552935:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574631.552941:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574631.552949:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574631.552956:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574631.552960:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574631.552963:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574631.552966:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574631.808900:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309991662, at 4309991680 00000100:00000010:0.0:1788574631.808919:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574632.257003:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574632.257025:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574632.257029:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574632.576947:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574632.576957:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574632.576960:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574632.576966:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574632.576977:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574632.576984:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574632.576989:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574632.576997:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574632.577010:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574633.089256:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574633.089267:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574633.089274:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574633.089293:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516721792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574633.089321:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574633.089342:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574633.089350:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574633.089356:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574633.089363:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574633.089381:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516721792:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574633.089396:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574633.089406:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574633.089420:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574633.089427:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574633.089440:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574633.089447:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574633.089460:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574633.089472:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915528. 00000400:00000010:0.0:1788574633.089483:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9ee0. 00000100:00000200:0.0:1788574633.089495:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516721792, portal 25 00000100:00000001:0.0:1788574633.089500:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574633.089502:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788574633.089515:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516721792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574633.089566:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574633.089579:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574633.089587:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574633.089595:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516721792, offset 0 00000400:00000200:0.0:1788574633.089606:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574633.089622:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574633.089636:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574633.089648:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574633.089658:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574633.089668:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574633.089680:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574633.089696:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547291:547291:256:4294967295] 192.168.202.128@tcp LPNI seq info [547214:547214:8:4294967295] 00000400:00000200:0.0:1788574633.089716:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574633.089745:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574633.089773:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574633.089785:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574633.089805:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788574633.089897:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574633.089918:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574633.089927:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4309996782 00000100:00000010:0.0:1788574633.089943:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574633.089953:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574633.089963:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4309996782 00000100:00000010:0.0:1788574633.089972:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50ba0. 00000100:00000001:0.0:1788574633.089981:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574633.089987:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574633.089994:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574633.090001:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574633.090013:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574633.090025:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574633.090037:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788574633.090044:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574633.090052:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574633.090057:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574633.090085:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516721920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574633.090121:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574633.090123:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574633.090128:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574633.090135:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574633.090139:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574633.090151:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516721920:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574633.090155:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574633.090161:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574633.090166:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574633.090170:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574633.090174:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574633.090181:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574633.090187:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574633.090199:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915790. 00000400:00000010:0.0:1788574633.090206:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9a18. 00000100:00000200:0.0:1788574633.090217:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516721920, portal 10 00000100:00000001:0.0:1788574633.090225:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574633.090229:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788574633.090243:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516721920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574633.090266:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574633.090276:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574633.090284:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9d48. 00000100:00000200:0.0:1788574633.090291:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516721920, offset 0 00000400:00000200:0.0:1788574633.090312:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574633.090321:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574633.090459:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574633.090473:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574633.090477:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574633.090479:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574633.090482:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574633.090486:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547292:547292:255:4294967295] 192.168.202.128@tcp LPNI seq info [547215:547215:7:4294967295] 00000400:00000200:0.0:1788574633.090491:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574633.090499:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574633.090504:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574633.090509:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56800. 00000800:00000200:0.0:1788574633.090514:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788574633.090522:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574633.090527:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574633.090531:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574633.090533:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309992979 00000100:00000001:0.0:1788574633.090535:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309996782 00000100:00000001:0.0:1788574633.090538:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309992979 00000100:00000001:0.0:1788574633.090540:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309996782 00000100:00000001:0.0:1788574633.090543:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574633.090545:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574633.090547:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574633.090554:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516722048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574633.090562:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574633.090563:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574633.090566:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574633.090568:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574633.090570:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574633.090574:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516722048:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574633.090577:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574633.090586:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574633.090588:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574633.090590:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574633.090594:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574633.090596:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574633.090600:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574633.090603:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b58. 00000400:00000010:0.0:1788574633.090607:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9908. 00000100:00000200:0.0:1788574633.090611:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516722048, portal 10 00000100:00000001:0.0:1788574633.090613:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574633.090615:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788574633.090622:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516722048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574633.090630:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574633.090632:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574633.090635:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9c38. 00000100:00000200:0.0:1788574633.090638:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516722048, offset 0 00000400:00000200:0.0:1788574633.090644:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574633.090648:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574633.090652:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574633.090655:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574633.090658:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574633.090661:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574633.090663:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574633.090667:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547293:547293:254:4294967295] 192.168.202.128@tcp LPNI seq info [547216:547216:6:4294967295] 00000400:00000200:0.0:1788574633.090672:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574633.090679:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574633.090684:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574633.090688:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56100. 00000800:00000200:0.0:1788574633.090693:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788574633.090700:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574633.090704:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574633.090713:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574633.090716:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309992986 00000100:00000001:0.0:1788574633.090718:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309996782 00000100:00000001:0.0:1788574633.090720:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309992986 00000100:00000001:0.0:1788574633.090723:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309996782 00000100:00000001:0.0:1788574633.090725:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574633.090728:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574633.090730:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574633.090733:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574633.090737:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574633.090739:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788574633.090742:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574633.090744:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574633.090745:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574633.090752:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516722176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574633.090760:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574633.090762:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574633.090765:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574633.090767:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574633.090770:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574633.090773:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516722176:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574633.090776:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574633.090779:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574633.090781:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574633.090782:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574633.090786:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574633.090789:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574633.090793:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574633.090796:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915948. 00000400:00000010:0.0:1788574633.090801:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9330. 00000100:00000200:0.0:1788574633.090804:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516722176, portal 4 00000100:00000001:0.0:1788574633.090841:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574633.090842:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788574633.090850:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516722176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574633.090858:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574633.090860:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574633.090863:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574633.090866:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516722176, offset 0 00000400:00000200:0.0:1788574633.090871:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574633.090876:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574633.090880:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574633.090883:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574633.090886:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574633.090889:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574633.090891:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574633.090894:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547294:547294:253:4294967295] 192.168.202.128@tcp LPNI seq info [547217:547217:5:4294967295] 00000400:00000200:0.0:1788574633.090899:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574633.090905:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574633.090910:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574633.090915:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56f00. 00000800:00000200:0.0:1788574633.090919:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788574633.090926:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574633.090929:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574633.090933:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574633.090934:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309993127 00000100:00000001:0.0:1788574633.090937:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309996782 00000100:00000001:0.0:1788574633.090940:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309993127 00000100:00000001:0.0:1788574633.090942:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309996782 00000100:00000001:0.0:1788574633.090945:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574633.090953:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574633.090955:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574633.090960:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516722304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574633.090969:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574633.090971:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574633.090973:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574633.090975:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574633.090978:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574633.090981:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516722304:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574633.090985:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574633.090987:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574633.090989:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574633.090991:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574633.090994:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574633.090996:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574633.091000:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574633.091002:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19152c0. 00000400:00000010:0.0:1788574633.091006:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed95d8. 00000100:00000200:0.0:1788574633.091010:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516722304, portal 4 00000100:00000001:0.0:1788574633.091012:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574633.091013:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788574633.091020:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516722304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574633.091027:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574633.091029:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574633.091032:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574633.091034:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516722304, offset 0 00000400:00000200:0.0:1788574633.091039:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574633.091043:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574633.091046:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574633.091049:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574633.091060:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574633.091063:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574633.091065:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574633.091068:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547295:547295:252:4294967295] 192.168.202.128@tcp LPNI seq info [547218:547218:4:4294967295] 00000400:00000200:0.0:1788574633.091073:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574633.091081:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574633.091086:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574633.091090:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574633.091094:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788574633.091101:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574633.091104:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574633.091108:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574633.091110:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309993129 00000100:00000001:0.0:1788574633.091113:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4309996782 00000100:00000001:0.0:1788574633.091115:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309993129 00000100:00000001:0.0:1788574633.091117:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4309996782 00000100:00000001:0.0:1788574633.091120:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574633.091122:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574633.091126:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574633.091128:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574633.091132:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574633.091134:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788574633.091136:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574633.091138:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574633.091139:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574633.091142:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574633.600989:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574633.600995:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574633.601004:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574633.601015:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574633.601021:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574633.601026:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574633.601061:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574633.601067:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574633.601076:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574634.305079:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574634.305085:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574634.305089:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574634.624955:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574634.624963:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574634.624970:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574634.624989:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574634.625011:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574634.625022:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574634.625043:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574634.625046:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574634.625050:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574634.625077:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574634.625081:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574634.625083:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574634.625085:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574634.625087:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574634.625090:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574634.625094:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574634.625096:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574634.625098:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574635.648970:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574635.648983:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574635.648990:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574635.649001:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574635.649010:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574635.649017:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574635.649050:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574635.649053:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574635.649056:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574635.649072:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574635.649075:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574635.649077:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574635.649079:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574635.649082:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574635.649108:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574635.649110:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574635.649111:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574635.649112:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788574636.161180:0:2389:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788574636.161228:0:2389:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788574636.161238:0:2389:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574636.161241:0:2389:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788574636.161246:0:2389:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574636.161250:0:2389:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788574636.161255:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574636.161259:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56400. 00000400:00000200:0.0:1788574636.161264:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574636.161268:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574636.161275:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574636.161278:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574636.161284:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574636.161307:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac820000 x1875440516721792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574636.161338:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574636.161344:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516721792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574636.161353:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574636.161356:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574636.161369:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574636.161372:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56800. 00000400:00000200:0.0:1788574636.161377:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574636.161381:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574636.161386:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9d48 00000400:00000010:0.0:1788574636.161388:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9d48. 00000100:00000001:0.0:1788574636.161392:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574636.161399:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac823b80 x1875440516721920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574636.161457:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574636.161465:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516721920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574636.161475:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574636.161477:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574636.161486:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574636.161494:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56100. 00000400:00000200:0.0:1788574636.161499:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574636.161507:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574636.161513:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9c38 00000400:00000010:0.0:1788574636.161519:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9c38. 00000100:00000001:0.0:1788574636.161526:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574636.161536:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac821500 x1875440516722048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574636.161556:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574636.161565:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516722048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574636.161578:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574636.161581:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574636.161593:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574636.161600:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56f00. 00000400:00000200:0.0:1788574636.161608:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574636.161617:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574636.161622:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574636.161626:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574636.161633:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574636.161642:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac823100 x1875440516722176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574636.161663:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574636.161670:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516722176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574636.161689:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574636.161693:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788574636.161705:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788574636.161711:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574636.161717:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788574636.161727:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788574636.161734:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574636.161739:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574636.161742:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574636.161751:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac822a00 x1875440516722304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574636.161768:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574636.161777:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516722304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574658 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574636.161791:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574636.161795:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788574636.353120:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574636.353125:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574636.353128:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574636.672945:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574636.672954:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574636.672960:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574636.672967:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574636.672975:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574636.672981:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574636.673013:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574636.673022:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574636.673031:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574636.928975:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4309996782, at 4309996800 00000100:00000010:0.0:1788574636.929008:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574637.376990:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574637.376993:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574637.377012:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574637.696919:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574637.696927:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574637.696930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574637.696936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574637.696943:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574637.696949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574637.696966:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574637.696969:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574637.696971:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.209200:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574638.209207:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574638.209212:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574638.209223:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516722432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.209240:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574638.209245:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574638.209253:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574638.209259:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574638.209283:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574638.209291:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516722432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574638.209299:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574638.209306:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574638.209312:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574638.209318:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574638.209326:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574638.209332:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574638.209343:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574638.209349:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574638.209360:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574638.209371:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516722432, portal 25 00000100:00000001:0.0:1788574638.209377:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574638.209381:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788574638.209395:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516722432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.209441:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574638.209449:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574638.209455:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574638.209462:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516722432, offset 0 00000400:00000200:0.0:1788574638.209475:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574638.209488:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574638.209505:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574638.209515:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574638.209524:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574638.209531:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574638.209538:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574638.209546:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547296:547296:256:4294967295] 192.168.202.128@tcp LPNI seq info [547219:547219:8:4294967295] 00000400:00000200:0.0:1788574638.209557:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574638.209579:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574638.209598:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574638.209607:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574638.209620:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574638.209629:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574638.209659:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574638.209664:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574638.209673:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574638.209686:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574638.209701:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574638.209706:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574638.209715:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574638.209725:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516722432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.209747:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574638.209755:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516722432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.209787:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.209794:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574638.209805:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574638.209839:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574638.209842:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310001902 00000100:00000010:0.0:1788574638.209848:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574638.209856:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574638.209867:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310001902 00000100:00000010:0.0:1788574638.209874:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b500c0. 00000100:00000001:0.0:1788574638.209881:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574638.209885:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574638.209890:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.209894:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574638.209904:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574638/real 1788574638] req@ffff8fc4ac821500 x1875440516722432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574638.209927:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574638.209931:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574638.209935:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574638.209940:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574638.209945:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574638.209951:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574638.209960:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516722432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574638.209981:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516722432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210001:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574638.210005:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.210011:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574638.210016:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574638.210026:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516722432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210053:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574638.210062:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516722432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210075:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574638.210082:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.210086:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574638.210091:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.210095:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574638.210101:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574638.210107:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574638.210118:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574638.210124:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574638.210135:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516722432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574638.210151:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516722432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574638.210160:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.210166:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574638.210173:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516722432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210186:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574638.210190:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574638.210195:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574638.210201:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574638.210204:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574638.210209:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574638.210216:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574638.210221:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788574638.210227:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574638.210237:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574638.210242:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788574638.210248:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574638.210251:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.210255:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.210264:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574638.210266:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.210270:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574638.210283:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574638.210286:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574638.210293:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516722560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210300:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574638.210302:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574638.210306:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574638.210308:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574638.210310:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574638.210314:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516722560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574638.210318:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574638.210321:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574638.210323:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574638.210325:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574638.210329:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574638.210331:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574638.210336:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574638.210343:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574638.210348:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574638.210353:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516722560, portal 10 00000100:00000001:0.0:1788574638.210355:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574638.210356:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788574638.210364:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516722560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210370:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574638.210373:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574638.210381:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574638.210384:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516722560, offset 0 00000400:00000200:0.0:1788574638.210388:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574638.210393:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574638.210398:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574638.210402:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574638.210405:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574638.210408:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574638.210410:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574638.210414:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547297:547297:256:4294967295] 192.168.202.128@tcp LPNI seq info [547220:547220:8:4294967295] 00000400:00000200:0.0:1788574638.210418:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574638.210425:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574638.210429:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574638.210432:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574638.210437:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574638.210440:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574638.210460:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574638.210462:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574638.210466:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574638.210469:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574638.210472:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574638.210474:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574638.210478:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574638.210481:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516722560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210489:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574638.210492:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516722560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210499:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.210501:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574638.210511:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574638.210516:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574638.210520:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574638.210522:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309998099 00000100:00000001:0.0:1788574638.210525:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310001902 00000100:00000001:0.0:1788574638.210528:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309998099 00000100:00000001:0.0:1788574638.210531:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310001902 00000100:00000001:0.0:1788574638.210533:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574638.210535:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.210537:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574638.210541:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574638/real 1788574638] req@ffff8fc4ac823b80 x1875440516722560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574638.210550:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574638.210551:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574638.210553:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574638.210556:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574638.210558:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574638.210560:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574638.210563:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516722560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574638.210573:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516722560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210580:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574638.210581:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.210583:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574638.210585:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574638.210588:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516722560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210596:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574638.210600:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516722560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210613:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574638.210616:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.210618:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574638.210619:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.210621:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574638.210624:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574638.210626:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574638.210630:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574638.210632:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574638.210637:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516722560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574638.210645:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516722560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574638.210649:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574638.210653:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516722688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210660:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574638.210662:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574638.210664:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574638.210666:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574638.210668:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574638.210671:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516722688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574638.210675:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574638.210678:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574638.210679:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574638.210681:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574638.210686:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574638.210687:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574638.210697:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574638.210700:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574638.210704:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574638.210707:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516722688, portal 10 00000100:00000001:0.0:1788574638.210709:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574638.210711:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788574638.210717:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516722688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210729:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574638.210731:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574638.210734:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574638.210736:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516722688, offset 0 00000400:00000200:0.0:1788574638.210741:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574638.210745:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574638.210749:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574638.210753:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574638.210755:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574638.210757:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574638.210759:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574638.210762:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547298:547298:256:4294967295] 192.168.202.128@tcp LPNI seq info [547221:547221:8:4294967295] 00000400:00000200:0.0:1788574638.210766:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574638.210772:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574638.210776:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574638.210780:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574638.210784:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574638.210787:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574638.210797:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574638.210798:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574638.210802:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574638.210805:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574638.210836:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574638.210846:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574638.210851:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574638.210856:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516722688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210863:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574638.210866:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516722688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210872:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.210873:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574638.210877:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574638.210880:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574638.210884:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574638.210886:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309998107 00000100:00000001:0.0:1788574638.210888:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310001902 00000100:00000001:0.0:1788574638.210890:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309998107 00000100:00000001:0.0:1788574638.210893:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310001902 00000100:00000001:0.0:1788574638.210895:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574638.210897:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.210899:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574638.210903:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574638/real 1788574638] req@ffff8fc4ac822a00 x1875440516722688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574638.210911:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574638.210912:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574638.210914:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574638.210916:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574638.210918:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574638.210920:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574638.210923:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516722688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574638.210931:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516722688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210942:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574638.210944:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.210946:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574638.210947:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574638.210951:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516722688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210957:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574638.210961:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516722688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.210968:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574638.210971:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.210972:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574638.210974:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.210976:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574638.210979:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574638.210981:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574638.210984:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574638.210987:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574638.210990:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516722688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574638.210998:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516722688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574638.211003:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574638.211006:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516722816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211013:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574638.211014:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574638.211023:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574638.211025:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574638.211027:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574638.211031:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516722816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574638.211034:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574638.211036:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574638.211038:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574638.211039:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574638.211043:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574638.211045:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574638.211049:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574638.211052:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574638.211056:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574638.211059:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516722816, portal 4 00000100:00000001:0.0:1788574638.211061:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574638.211063:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788574638.211068:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516722816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211074:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574638.211076:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574638.211079:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574638.211081:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516722816, offset 0 00000400:00000200:0.0:1788574638.211085:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574638.211090:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574638.211094:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574638.211097:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574638.211099:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574638.211101:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574638.211103:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574638.211106:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547299:547299:256:4294967295] 192.168.202.128@tcp LPNI seq info [547222:547222:8:4294967295] 00000400:00000200:0.0:1788574638.211110:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574638.211117:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574638.211128:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574638.211131:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574638.211136:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574638.211139:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574638.211150:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574638.211152:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574638.211155:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574638.211157:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574638.211160:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574638.211161:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574638.211163:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574638.211166:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516722816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211172:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574638.211174:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516722816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211178:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.211180:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574638.211182:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574638.211184:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574638.211187:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574638.211188:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309998247 00000100:00000001:0.0:1788574638.211190:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310001902 00000100:00000001:0.0:1788574638.211192:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309998247 00000100:00000001:0.0:1788574638.211193:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310001902 00000100:00000001:0.0:1788574638.211194:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574638.211195:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.211197:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574638.211199:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574638/real 1788574638] req@ffff8fc4ac823100 x1875440516722816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574638.211209:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574638.211210:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574638.211212:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574638.211213:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574638.211215:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574638.211216:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574638.211218:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516722816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574638.211224:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516722816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211228:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574638.211229:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.211230:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574638.211231:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574638.211233:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516722816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211237:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574638.211239:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516722816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211243:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574638.211245:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.211246:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574638.211247:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.211249:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574638.211250:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574638.211252:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574638.211254:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574638.211255:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574638.211261:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516722816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574638.211266:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516722816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574638.211269:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574638.211288:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516722944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211296:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574638.211297:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574638.211300:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574638.211301:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574638.211304:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574638.211310:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516722944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574638.211315:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574638.211317:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574638.211318:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574638.211320:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574638.211324:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574638.211325:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574638.211329:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574638.211332:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574638.211335:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574638.211338:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516722944, portal 4 00000100:00000001:0.0:1788574638.211340:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574638.211341:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788574638.211347:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516722944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211353:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574638.211355:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574638.211357:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574638.211359:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516722944, offset 0 00000400:00000200:0.0:1788574638.211371:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574638.211375:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574638.211379:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574638.211382:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574638.211384:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574638.211386:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574638.211388:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574638.211391:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547300:547300:256:4294967295] 192.168.202.128@tcp LPNI seq info [547223:547223:8:4294967295] 00000400:00000200:0.0:1788574638.211395:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574638.211401:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574638.211406:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574638.211410:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574638.211414:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574638.211416:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574638.211426:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574638.211428:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574638.211431:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574638.211434:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574638.211437:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574638.211439:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574638.211442:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574638.211445:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516722944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211453:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574638.211455:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516722944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211462:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.211463:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574638.211466:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574638.211469:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574638.211472:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574638.211483:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4309998249 00000100:00000001:0.0:1788574638.211485:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310001902 00000100:00000001:0.0:1788574638.211487:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4309998249 00000100:00000001:0.0:1788574638.211489:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310001902 00000100:00000001:0.0:1788574638.211491:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574638.211493:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.211495:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574638.211500:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574638/real 1788574638] req@ffff8fc4ac820000 x1875440516722944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574638.211508:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574638.211510:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574638.211512:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574638.211514:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574638.211516:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574638.211519:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574638.211523:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516722944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574638.211530:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516722944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211536:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574638.211538:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.211540:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574638.211541:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574638.211544:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516722944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211550:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574638.211553:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516722944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211559:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574638.211567:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.211568:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574638.211569:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.211571:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574638.211573:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574638.211576:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574638.211579:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574638.211581:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574638.211585:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516722944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574638.211593:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516722944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574638.211596:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.211599:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574638.211603:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516722560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211610:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574638.211612:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574638.211614:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574638.211617:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574638.211618:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574638.211620:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574638.211622:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574638.211624:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788574638.211627:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574638.211629:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574638.211631:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788574638.211634:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574638.211635:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.211637:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574638.211639:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516722688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211653:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574638.211654:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574638.211655:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574638.211657:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574638.211658:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574638.211658:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574638.211660:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574638.211661:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788574638.211663:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574638.211663:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574638.211664:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788574638.211665:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574638.211666:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.211667:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574638.211670:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516722816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211674:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574638.211675:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574638.211676:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574638.211677:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574638.211678:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574638.211679:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574638.211680:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574638.211681:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788574638.211682:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574638.211683:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574638.211684:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788574638.211685:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574638.211686:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.211687:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574638.211689:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516722944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574663 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574638.211692:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574638.211693:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574638.211697:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574638.211699:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574638.211700:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574638.211701:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574638.211702:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574638.211703:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788574638.211704:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574638.211705:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574638.211706:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788574638.211707:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574638.211708:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.211709:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.211714:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574638.211715:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.211716:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574638.211717:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.720909:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574638.720916:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574638.720920:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574638.720926:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574638.720931:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574638.720936:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.720947:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574638.720951:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.720956:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574638.720997:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574638.721003:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574638.721007:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574638.721010:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574638.721014:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574638.721017:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.721020:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574638.721022:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574638.721025:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574639.424965:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574639.424968:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574639.424971:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574639.744956:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574639.744973:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574639.744977:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574639.744980:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574639.744985:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574639.744989:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574639.744999:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574639.745004:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574639.745009:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574640.768886:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574640.768892:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574640.768895:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574640.768902:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574640.768908:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574640.768913:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574640.768917:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574640.768921:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574640.768925:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574641.472964:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574641.472969:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574641.472972:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574641.793009:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574641.793017:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574641.793022:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574641.793031:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574641.793039:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574641.793045:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574641.793054:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574641.793065:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574641.793084:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574642.048954:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310001902, at 4310001920 00000100:00000010:0.0:1788574642.048973:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574642.497041:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574642.497047:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574642.497051:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574642.816907:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574642.816915:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574642.816920:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574642.816945:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574642.816951:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574642.816956:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574642.816978:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574642.816983:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574642.816989:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.329312:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574643.329323:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574643.329327:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574643.329341:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516723072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.329359:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574643.329366:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574643.329371:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574643.329374:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574643.329376:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574643.329383:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516723072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574643.329390:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574643.329397:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574643.329401:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574643.329405:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574643.329417:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574643.329425:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574643.329440:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574643.329450:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574643.329462:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574643.329472:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516723072, portal 25 00000100:00000001:0.0:1788574643.329477:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574643.329478:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788574643.329487:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516723072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.329499:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574643.329507:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574643.329515:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574643.329542:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516723072, offset 0 00000400:00000200:0.0:1788574643.329558:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574643.329573:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574643.329580:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574643.329587:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574643.329594:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574643.329598:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574643.329604:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574643.329610:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547301:547301:256:4294967295] 192.168.202.128@tcp LPNI seq info [547224:547224:8:4294967295] 00000400:00000200:0.0:1788574643.329614:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574643.329624:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574643.329634:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574643.329645:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574643.329655:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574643.329664:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574643.329709:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574643.329718:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574643.329724:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574643.329734:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574643.329739:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574643.329751:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574643.329757:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574643.329766:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516723072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.329793:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574643.329834:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516723072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.329843:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.329845:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574643.329852:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574643.329860:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574643.329871:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310007022 00000100:00000010:0.0:1788574643.329881:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574643.329891:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574643.329904:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310007022 00000100:00000010:0.0:1788574643.329916:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50840. 00000100:00000001:0.0:1788574643.329924:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574643.329935:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574643.329941:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.329947:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574643.329961:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574643/real 1788574643] req@ffff8fc4ac822d80 x1875440516723072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574643.329984:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574643.329991:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574643.329997:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574643.330003:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574643.330006:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574643.330012:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574643.330020:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516723072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574643.330037:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516723072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.330056:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574643.330059:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.330065:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574643.330067:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574643.330073:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516723072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.330089:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574643.330097:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516723072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.330158:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574643.330170:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.330178:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574643.330184:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.330191:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574643.330195:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574643.330200:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574643.330213:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574643.330234:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574643.330245:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516723072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574643.330252:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516723072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574643.330266:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.330271:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574643.330277:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516723072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.330300:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574643.330304:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574643.330307:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574643.330312:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574643.330317:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574643.330323:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574643.330327:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574643.330331:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788574643.330343:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574643.330346:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574643.330352:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788574643.330362:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574643.330364:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.330369:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.330384:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574643.330396:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.330402:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574643.330406:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574643.330411:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574643.330421:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516723200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.330459:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574643.330463:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574643.330472:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574643.330479:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574643.330482:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574643.330490:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516723200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574643.330506:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574643.330516:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574643.330519:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574643.330522:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574643.330526:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574643.330530:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574643.330542:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574643.330547:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574643.330551:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574643.330554:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516723200, portal 10 00000100:00000001:0.0:1788574643.330556:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574643.330557:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788574643.330570:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516723200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.330590:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574643.330594:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574643.330598:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574643.330604:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516723200, offset 0 00000400:00000200:0.0:1788574643.330609:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574643.330618:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574643.330631:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574643.330637:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574643.330648:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574643.330658:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574643.330668:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574643.330675:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547302:547302:256:4294967295] 192.168.202.128@tcp LPNI seq info [547225:547225:8:4294967295] 00000400:00000200:0.0:1788574643.330686:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574643.330703:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574643.330719:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574643.330728:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574643.330740:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574643.330744:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574643.330775:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574643.330783:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574643.330789:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574643.330805:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574643.330836:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574643.330844:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574643.330850:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574643.330860:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516723200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.330881:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574643.330890:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516723200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.330912:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.330915:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574643.330925:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574643.330936:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574643.330949:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574643.330958:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310003220 00000100:00000001:0.0:1788574643.330975:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310007022 00000100:00000001:0.0:1788574643.330989:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310003220 00000100:00000001:0.0:1788574643.331000:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310007022 00000100:00000001:0.0:1788574643.331004:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574643.331014:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.331017:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574643.331031:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574643/real 1788574643] req@ffff8fc4ac821f80 x1875440516723200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574643.331050:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574643.331054:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574643.331059:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574643.331067:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574643.331071:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574643.331079:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574643.331085:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516723200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574643.331103:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516723200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331111:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574643.331115:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331119:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574643.331129:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574643.331140:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516723200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331156:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574643.331164:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516723200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331184:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574643.331193:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331201:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574643.331205:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331212:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574643.331219:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574643.331241:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574643.331254:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574643.331264:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574643.331277:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516723200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574643.331291:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516723200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574643.331295:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574643.331299:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516723328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331306:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574643.331308:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574643.331310:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574643.331313:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574643.331315:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574643.331317:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516723328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574643.331321:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574643.331323:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574643.331326:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574643.331327:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574643.331330:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574643.331332:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574643.331335:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574643.331337:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574643.331340:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574643.331342:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516723328, portal 10 00000100:00000001:0.0:1788574643.331344:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574643.331353:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788574643.331360:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516723328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331368:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574643.331370:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574643.331373:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574643.331376:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516723328, offset 0 00000400:00000200:0.0:1788574643.331381:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574643.331385:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574643.331389:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574643.331392:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574643.331395:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574643.331398:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574643.331400:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574643.331404:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547303:547303:256:4294967295] 192.168.202.128@tcp LPNI seq info [547226:547226:8:4294967295] 00000400:00000200:0.0:1788574643.331408:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574643.331415:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574643.331420:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574643.331424:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574643.331430:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574643.331433:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574643.331445:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574643.331447:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574643.331451:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574643.331454:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574643.331457:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574643.331459:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574643.331463:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574643.331466:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516723328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331486:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574643.331489:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516723328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331493:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.331494:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574643.331498:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574643.331501:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574643.331503:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574643.331505:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310003227 00000100:00000001:0.0:1788574643.331507:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310007022 00000100:00000001:0.0:1788574643.331508:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310003227 00000100:00000001:0.0:1788574643.331510:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310007022 00000100:00000001:0.0:1788574643.331511:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574643.331513:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.331514:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574643.331516:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574643/real 1788574643] req@ffff8fc4ac823480 x1875440516723328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574643.331522:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574643.331523:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574643.331524:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574643.331525:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574643.331527:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574643.331528:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574643.331530:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516723328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574643.331536:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516723328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331540:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574643.331544:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331546:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574643.331547:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574643.331549:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516723328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331554:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574643.331556:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516723328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331560:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574643.331562:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331563:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574643.331564:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331565:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574643.331568:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574643.331570:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574643.331572:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574643.331573:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574643.331576:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516723328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574643.331581:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516723328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574643.331583:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331585:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574643.331587:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516723200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331591:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574643.331592:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574643.331593:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574643.331595:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574643.331595:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574643.331599:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574643.331601:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574643.331602:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574643.331604:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574643.331605:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574643.331606:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788574643.331608:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574643.331609:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331610:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574643.331613:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516723328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331617:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574643.331618:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574643.331618:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574643.331620:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574643.331621:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574643.331622:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574643.331623:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574643.331624:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788574643.331625:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574643.331626:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574643.331627:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788574643.331628:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574643.331629:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331630:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331633:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574643.331634:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.331635:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574643.331636:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574643.331637:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574643.331640:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516723456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331645:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574643.331646:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574643.331648:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574643.331649:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574643.331653:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574643.331655:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516723456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574643.331658:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574643.331660:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574643.331661:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574643.331662:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574643.331664:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574643.331665:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574643.331667:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574643.331669:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574643.331672:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574643.331674:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516723456, portal 4 00000100:00000001:0.0:1788574643.331675:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574643.331676:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788574643.331680:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516723456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331685:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574643.331686:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574643.331688:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574643.331689:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516723456, offset 0 00000400:00000200:0.0:1788574643.331691:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574643.331694:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574643.331696:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574643.331698:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574643.331700:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574643.331701:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574643.331703:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574643.331705:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547304:547304:256:4294967295] 192.168.202.128@tcp LPNI seq info [547227:547227:8:4294967295] 00000400:00000200:0.0:1788574643.331708:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574643.331712:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574643.331715:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574643.331720:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574643.331723:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574643.331725:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574643.331733:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574643.331734:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574643.331736:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574643.331738:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574643.331741:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574643.331742:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574643.331744:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574643.331746:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516723456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331752:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574643.331754:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516723456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331758:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.331759:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574643.331761:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574643.331763:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574643.331765:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574643.331766:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310003367 00000100:00000001:0.0:1788574643.331768:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310007022 00000100:00000001:0.0:1788574643.331769:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310003367 00000100:00000001:0.0:1788574643.331771:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310007022 00000100:00000001:0.0:1788574643.331773:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574643.331775:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.331777:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574643.331780:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574643/real 1788574643] req@ffff8fc4ac822300 x1875440516723456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574643.331801:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574643.331803:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574643.331805:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574643.331831:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574643.331832:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574643.331834:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574643.331838:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516723456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574643.331844:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516723456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331848:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574643.331849:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331851:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574643.331852:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574643.331854:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516723456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331858:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574643.331861:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516723456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331866:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574643.331867:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331868:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574643.331869:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.331870:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574643.331872:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574643.331873:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574643.331875:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574643.331877:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574643.331885:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516723456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574643.331890:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516723456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574643.331893:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574643.331897:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516723584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331904:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574643.331906:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574643.331909:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574643.331911:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574643.331913:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574643.331916:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516723584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574643.331920:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574643.331922:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574643.331924:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574643.331926:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574643.331929:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574643.331932:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574643.331935:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574643.331938:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574643.331942:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574643.331945:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516723584, portal 4 00000100:00000001:0.0:1788574643.331947:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574643.331949:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788574643.331956:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516723584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.331963:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574643.331966:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574643.331968:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574643.331971:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516723584, offset 0 00000400:00000200:0.0:1788574643.331981:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574643.331984:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574643.331987:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574643.331989:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574643.331991:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574643.331992:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574643.331994:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574643.331996:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547305:547305:256:4294967295] 192.168.202.128@tcp LPNI seq info [547228:547228:8:4294967295] 00000400:00000200:0.0:1788574643.331999:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574643.332003:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574643.332006:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574643.332008:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574643.332011:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574643.332013:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574643.332020:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574643.332021:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574643.332024:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574643.332026:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574643.332028:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574643.332029:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574643.332031:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574643.332034:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516723584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.332039:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574643.332042:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516723584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574668 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.332046:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.332047:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574643.332049:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574643.332051:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574643.332053:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574643.332059:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310003370 00000100:00000001:0.0:1788574643.332060:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310007022 00000100:00000001:0.0:1788574643.332062:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310003370 00000100:00000001:0.0:1788574643.332063:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310007022 00000100:00000001:0.0:1788574643.332065:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574643.332066:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.332068:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574643.332070:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574643/real 1788574643] req@ffff8fc4ac820700 x1875440516723584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574643.332075:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574643.332076:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574643.332078:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574643.332079:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574643.332080:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574643.332082:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574643.332084:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516723584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574643.332089:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516723584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.332093:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574643.332094:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.332095:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574643.332096:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574643.332098:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516723584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.332102:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574643.332104:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516723584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.332109:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574643.332113:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.332114:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574643.332115:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.332117:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574643.332118:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574643.332120:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574643.332122:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574643.332124:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574643.332126:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516723584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574643.332131:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516723584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574643.332134:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.332135:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574643.332137:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516723456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.332141:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574643.332142:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574643.332143:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574643.332145:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574643.332146:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574643.332147:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574643.332148:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574643.332149:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788574643.332150:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574643.332151:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574643.332153:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788574643.332154:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574643.332155:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.332156:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574643.332158:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516723584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574643.332168:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574643.332169:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574643.332171:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574643.332172:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574643.332173:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574643.332174:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574643.332175:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574643.332175:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788574643.332177:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574643.332177:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574643.332178:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788574643.332180:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574643.332180:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.332181:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.332184:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574643.332185:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.332186:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574643.332187:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.840930:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574643.840940:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574643.840945:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574643.840952:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574643.840961:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574643.840967:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.840990:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574643.840994:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.841000:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574643.841043:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574643.841047:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574643.841049:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574643.841054:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574643.841060:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574643.841066:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.841071:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574643.841075:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574643.841078:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574644.545050:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574644.545055:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574644.545073:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574644.865107:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574644.865110:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574644.865111:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574644.865114:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574644.865118:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574644.865123:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574644.865130:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574644.865134:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574644.865141:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574645.888923:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574645.888931:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574645.888934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574645.888941:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574645.888946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574645.888951:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574645.888955:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574645.888958:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574645.888962:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574646.593012:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574646.593015:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574646.593018:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574646.912896:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574646.912902:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574646.912907:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574646.912916:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574646.912923:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574646.912930:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574646.912952:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574646.912956:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574646.912959:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574647.168916:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310007022, at 4310007040 00000100:00000010:0.0:1788574647.168936:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574647.617020:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574647.617023:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574647.617043:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574647.936931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574647.936936:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574647.936940:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574647.936947:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574647.936955:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574647.936960:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574647.936966:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574647.936970:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574647.936974:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574648.448899:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574648.448911:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574648.448914:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574648.448917:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574648.448922:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574648.448927:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15335 00000400:00000200:0.0:1788574648.448937:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574648.448943:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574648.448947:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574648.448950:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.448954:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574648.448956:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.448959:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574648.448962:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574648.448964:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574648.448967:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574648.448969:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574648.448972:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574648.448974:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574648.448977:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.448979:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574648.448981:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.448997:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574648.449001:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574648.449004:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449007:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574648.449010:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574648.449013:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574648.449016:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449018:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574648.449020:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574648.449027:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574648.449031:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574648.449035:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574648.449037:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449040:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574648.449043:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574648.449048:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574648.449054:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574648.449061:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574648.449070:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516723712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574648.449081:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574648.449085:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574648.449090:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516723712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574648.449112:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574648.449116:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574648.449120:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574648.449122:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574648.449124:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574648.449128:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574648.449131:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15335 00000400:00000200:0.0:1788574648.449141:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574648.449148:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574648.449151:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574648.449158:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449163:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574648.449167:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449181:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574648.449185:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574648.449188:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574648.449190:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574648.449193:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574648.449198:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574648.449202:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574648.449209:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449212:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574648.449217:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449219:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574648.449221:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574648.449225:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449228:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574648.449231:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574648.449236:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574648.449239:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449241:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574648.449246:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574648.449248:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449249:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574648.449253:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574648.449256:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574648.449262:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574648.449269:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449273:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574648.449276:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574648.449280:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574648.449288:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574648.449295:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574648.449303:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516723840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574648.449319:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574648.449323:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574648.449331:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516723840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574648.449346:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574648.449352:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574648.449357:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574648.449359:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574648.449362:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574648.449365:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574648.449368:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15335 00000400:00000200:0.0:1788574648.449372:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574648.449375:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574648.449379:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574648.449381:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449383:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574648.449388:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449390:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574648.449395:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574648.449396:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574648.449397:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574648.449398:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574648.449400:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574648.449410:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574648.449414:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449418:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574648.449423:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449426:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574648.449428:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574648.449432:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449435:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574648.449438:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574648.449441:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574648.449446:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449449:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574648.449452:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574648.449455:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449460:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574648.449463:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574648.449468:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574648.449470:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574648.449471:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449475:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574648.449478:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574648.449481:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574648.449488:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574648.449495:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574648.449501:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516723968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574648.449514:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574648.449519:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574648.449527:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516723968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574648.449538:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574648.449549:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574648.449553:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574648.449555:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574648.449558:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574648.449563:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574648.449568:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15335 00000400:00000200:0.0:1788574648.449571:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574648.449574:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574648.449578:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574648.449583:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449585:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574648.449590:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449594:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574648.449599:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574648.449601:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574648.449604:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574648.449607:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574648.449610:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574648.449612:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574648.449619:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449624:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574648.449628:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449633:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574648.449637:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574648.449644:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449647:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574648.449649:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574648.449654:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574648.449659:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449663:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574648.449672:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574648.449676:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449680:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574648.449684:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574648.449689:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574648.449692:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574648.449693:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449696:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574648.449698:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574648.449701:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574648.449705:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574648.449710:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574648.449717:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516724096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574648.449728:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574648.449731:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574648.449744:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516724096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574648.449774:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574648.449777:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574648.449780:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574648.449782:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574648.449786:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574648.449791:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574648.449794:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15335 00000400:00000200:0.0:1788574648.449800:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574648.449804:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574648.449831:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574648.449837:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449843:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574648.449848:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449861:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574648.449867:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574648.449872:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574648.449876:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574648.449880:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574648.449885:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574648.449889:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574648.449897:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449901:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574648.449905:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449910:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574648.449915:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574648.449919:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574648.449920:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574648.449922:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574648.449924:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574648.449929:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449931:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574648.449934:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574648.449940:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449943:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574648.449948:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574648.449954:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574648.449959:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574648.449963:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574648.449965:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574648.449968:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574648.449972:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574648.449978:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574648.449985:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574648.449992:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516724224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574648.450041:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574648.450044:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574648.450051:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516724224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574648.450061:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574648.450068:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15346) 00000100:00000001:0.0:1788574648.960984:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574648.960988:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574648.960989:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574648.960992:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574648.960995:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574648.960997:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574648.960998:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574648.961000:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574648.961001:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574649.664991:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574649.664993:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574649.664996:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574649.984898:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574649.984903:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574649.984907:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574649.984913:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574649.984918:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574649.984922:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574649.984934:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574649.984938:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574649.984943:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574651.008927:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574651.008934:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574651.008938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574651.008944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574651.008950:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574651.008955:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574651.008961:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574651.008972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574651.008977:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574651.712978:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574651.712982:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574651.712984:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574652.032909:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574652.032916:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574652.032920:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574652.032928:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574652.032935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574652.032942:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574652.032961:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574652.032964:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574652.032967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574652.288912:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310012141, at 4310012160 00000100:00000010:0.0:1788574652.288930:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab780. 00000020:00000001:0.0:1788574652.737002:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574652.737005:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574652.737008:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574653.056945:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574653.056951:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574653.056956:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574653.056961:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574653.056967:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574653.056970:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574653.056988:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574653.056992:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574653.056995:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574653.568899:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574653.568911:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574653.568914:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574653.568919:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574653.568925:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574653.568934:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15341 00000400:00000200:0.0:1788574653.568944:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574653.568969:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574653.568973:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574653.568977:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.568980:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574653.568983:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.568985:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574653.568990:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574653.568992:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574653.568995:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574653.568999:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574653.569001:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574653.569004:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574653.569007:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569009:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574653.569011:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569013:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574653.569017:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574653.569022:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569024:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574653.569028:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574653.569031:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574653.569035:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.569038:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574653.569040:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574653.569047:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574653.569052:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574653.569056:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574653.569057:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.569061:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574653.569064:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574653.569068:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574653.569076:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574653.569090:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574653.569101:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516724352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574653.569112:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574653.569130:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574653.569136:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516724352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574653.569159:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574653.569162:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574653.569166:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574653.569168:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574653.569170:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574653.569173:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574653.569176:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15341 00000400:00000200:0.0:1788574653.569182:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574653.569187:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574653.569190:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574653.569197:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569203:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574653.569210:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569214:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574653.569219:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574653.569222:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574653.569226:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574653.569231:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574653.569236:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574653.569243:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574653.569250:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569255:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574653.569269:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569273:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574653.569281:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574653.569288:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569295:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574653.569300:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574653.569318:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574653.569325:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.569330:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574653.569336:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574653.569344:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.569348:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574653.569353:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574653.569358:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574653.569364:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574653.569368:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.569372:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574653.569374:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574653.569378:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574653.569386:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574653.569394:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574653.569405:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516724480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574653.569428:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574653.569431:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574653.569441:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516724480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574653.569460:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574653.569465:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574653.569470:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574653.569471:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574653.569481:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574653.569489:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574653.569505:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15341 00000400:00000200:0.0:1788574653.569515:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574653.569521:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574653.569524:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574653.569532:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569538:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574653.569544:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569547:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574653.569550:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574653.569552:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574653.569556:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574653.569558:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574653.569563:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574653.569569:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574653.569586:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569593:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574653.569597:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569602:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574653.569608:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574653.569614:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569620:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574653.569624:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574653.569631:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574653.569638:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.569643:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574653.569649:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574653.569656:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.569661:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574653.569675:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574653.569683:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574653.569690:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574653.569694:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.569698:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574653.569702:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574653.569706:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574653.569715:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574653.569721:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574653.569731:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516724608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574653.569753:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574653.569757:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574653.569764:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516724608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574653.569784:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574653.569789:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574653.569796:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574653.569801:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574653.569805:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574653.569841:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574653.569848:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15341 00000400:00000200:0.0:1788574653.569857:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574653.569865:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574653.569871:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574653.569879:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569885:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574653.569892:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569896:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574653.569898:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574653.569910:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574653.569915:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574653.569919:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574653.569924:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574653.569930:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574653.569939:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569944:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574653.569951:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569957:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574653.569964:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574653.569969:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574653.569975:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574653.569977:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574653.569981:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574653.569986:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.569991:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574653.569996:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574653.570002:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.570006:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574653.570015:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574653.570021:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574653.570028:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574653.570032:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.570037:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574653.570041:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574653.570047:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574653.570055:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574653.570063:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574653.570075:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516724736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574653.570090:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574653.570096:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574653.570131:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516724736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574653.570152:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574653.570159:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574653.570168:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574653.570173:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574653.570176:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574653.570182:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574653.570189:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15341 00000400:00000200:0.0:1788574653.570199:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574653.570208:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574653.570215:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574653.570223:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.570228:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574653.570234:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.570240:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574653.570245:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574653.570249:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574653.570254:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574653.570259:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574653.570263:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574653.570270:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574653.570279:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.570285:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574653.570292:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574653.570298:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574653.570304:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574653.570313:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574653.570320:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574653.570330:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574653.570335:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574653.570340:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.570346:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574653.570351:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574653.570358:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.570363:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574653.570370:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574653.570375:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574653.570381:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574653.570386:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574653.570391:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574653.570396:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574653.570400:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574653.570409:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574653.570415:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574653.570426:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516724864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574653.570456:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574653.570461:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574653.570472:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516724864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574653.570481:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574653.570486:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15351) 00000100:00000001:0.0:1788574654.080933:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574654.080939:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574654.080943:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574654.080949:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574654.080955:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574654.080959:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574654.080973:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574654.080976:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574654.080979:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574654.785060:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574654.785070:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574654.785073:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574655.104920:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574655.104930:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574655.104934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574655.104943:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574655.104949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574655.104954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574655.104962:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574655.104967:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574655.104970:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574655.105001:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574655.105004:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574655.105005:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574655.105007:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574655.105009:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574655.105011:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574655.105013:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574655.105014:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574655.105016:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574656.128917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574656.128924:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574656.128928:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574656.128937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574656.128943:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574656.128947:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574656.128953:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574656.128957:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574656.128963:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574656.832992:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574656.832996:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574656.832998:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574657.152929:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574657.152936:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574657.152941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574657.152950:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574657.152957:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574657.152963:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574657.152993:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574657.152996:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574657.152999:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574657.408881:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310017261, at 4310017280 00000100:00000010:0.0:1788574657.408895:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abf60. 00000020:00000001:0.0:1788574657.856990:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574657.856994:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574657.856997:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574658.176931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574658.176938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574658.176943:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574658.176949:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574658.176955:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574658.176959:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574658.176967:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574658.176971:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574658.176974:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574658.688936:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574658.688947:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574658.688949:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574658.688953:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574658.688957:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574658.688965:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15346 00000400:00000200:0.0:1788574658.688975:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574658.688981:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574658.688984:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574658.688987:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.688989:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574658.688991:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.688993:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574658.688996:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574658.688997:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574658.688999:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574658.689018:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574658.689020:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574658.689022:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574658.689024:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689026:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574658.689027:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689028:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574658.689031:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574658.689034:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689036:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574658.689039:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574658.689043:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574658.689046:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689049:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574658.689051:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574658.689058:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574658.689062:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574658.689083:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574658.689085:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689088:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574658.689091:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574658.689096:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574658.689105:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574658.689112:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574658.689122:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516724992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574658.689133:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574658.689138:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574658.689143:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516724992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574658.689165:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574658.689168:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574658.689176:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574658.689177:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574658.689178:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574658.689180:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574658.689183:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15346 00000400:00000200:0.0:1788574658.689187:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574658.689190:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574658.689192:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574658.689194:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689196:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574658.689198:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689202:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574658.689204:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574658.689207:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574658.689209:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574658.689212:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574658.689214:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574658.689217:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574658.689221:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689226:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574658.689230:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689233:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574658.689237:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574658.689241:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689244:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574658.689247:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574658.689252:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574658.689255:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689259:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574658.689263:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574658.689274:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689278:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574658.689282:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574658.689286:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574658.689290:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574658.689292:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689295:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574658.689297:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574658.689299:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574658.689306:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574658.689309:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574658.689316:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516725120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574658.689330:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574658.689333:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574658.689341:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516725120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574658.689352:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574658.689357:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574658.689360:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574658.689363:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574658.689366:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574658.689370:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574658.689375:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15346 00000400:00000200:0.0:1788574658.689381:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574658.689385:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574658.689388:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574658.689390:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689395:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574658.689400:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689410:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574658.689413:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574658.689416:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574658.689419:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574658.689422:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574658.689425:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574658.689430:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574658.689437:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689440:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574658.689445:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689447:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574658.689449:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574658.689454:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689459:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574658.689462:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574658.689466:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574658.689470:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689475:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574658.689477:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574658.689480:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689483:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574658.689490:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574658.689496:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574658.689503:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574658.689507:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689512:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574658.689517:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574658.689521:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574658.689529:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574658.689536:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574658.689545:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516725248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574658.689594:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574658.689598:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574658.689604:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516725248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574658.689623:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574658.689628:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574658.689634:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574658.689639:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574658.689643:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574658.689649:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574658.689654:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15346 00000400:00000200:0.0:1788574658.689659:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574658.689664:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574658.689667:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574658.689671:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689676:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574658.689679:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689681:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574658.689685:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574658.689686:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574658.689688:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574658.689691:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574658.689694:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574658.689700:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574658.689704:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689707:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574658.689711:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689712:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574658.689722:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574658.689728:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689730:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574658.689734:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574658.689738:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574658.689742:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689744:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574658.689748:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574658.689752:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689755:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574658.689759:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574658.689763:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574658.689767:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574658.689769:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689774:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574658.689775:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574658.689777:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574658.689784:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574658.689789:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574658.689797:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516725376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574658.689805:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574658.689826:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574658.689829:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516725376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574658.689835:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574658.689837:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574658.689840:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574658.689841:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574658.689842:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574658.689844:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574658.689853:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15346 00000400:00000200:0.0:1788574658.689856:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574658.689862:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574658.689867:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574658.689873:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689875:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574658.689879:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689882:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574658.689888:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574658.689890:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574658.689893:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574658.689896:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574658.689899:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574658.689902:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574658.689907:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689911:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574658.689915:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689918:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574658.689922:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574658.689924:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574658.689929:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574658.689932:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574658.689935:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574658.689937:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689940:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574658.689944:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574658.689947:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689951:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574658.689955:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574658.689958:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574658.689962:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574658.689969:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574658.689973:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574658.689976:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574658.689978:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574658.689985:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574658.689990:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574658.689999:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516725504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574658.690007:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574658.690011:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574658.690017:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516725504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574658.690034:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574658.690040:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15356) 00000100:00000001:0.0:1788574659.200921:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574659.200928:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574659.200933:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574659.200944:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574659.200952:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574659.200961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574659.200980:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574659.200986:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574659.200992:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574659.904991:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574659.904996:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574659.904999:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574661.248945:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574661.248951:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574661.248956:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574661.248962:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574661.248965:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574661.248970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574661.249009:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574661.249012:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574661.249016:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574661.249060:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574661.249063:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574661.249064:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574661.249066:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574661.249069:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574661.249071:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574661.249073:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574661.249074:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574661.249076:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574661.952991:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574661.952995:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574661.952998:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574662.273108:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574662.273140:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574662.273141:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574662.273145:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574662.273147:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574662.273149:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574662.273152:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574662.273153:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574662.273156:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574662.528907:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310022381, at 4310022400 00000100:00000010:0.0:1788574662.528920:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab9c0. 00000020:00000001:0.0:1788574662.976982:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574662.976986:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574662.976989:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574663.296925:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574663.296933:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574663.296938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574663.296946:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574663.296954:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574663.296959:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574663.296976:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574663.296981:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574663.296986:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574664.320910:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574664.320917:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574664.320937:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574664.320941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574664.320947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574664.320954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574664.320970:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574664.320974:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574664.320978:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574665.025033:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574665.025037:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574665.025039:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574665.344913:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574665.344921:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574665.344925:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574665.344932:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574665.344936:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574665.344943:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574665.344950:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574665.344954:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574665.344960:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574665.345013:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574665.345018:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574665.345020:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574665.345022:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574665.345026:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574665.345029:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574665.345035:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574665.345037:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574665.345041:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574666.368921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574666.368927:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574666.368932:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574666.368939:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574666.368946:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574666.368950:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574666.368964:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574666.368968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574666.368971:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574667.073094:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574667.073103:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574667.073105:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574667.392971:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574667.392988:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574667.392992:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574667.392998:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574667.393005:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574667.393011:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574667.393017:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574667.393020:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574667.393023:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574667.648925:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310027501, at 4310027520 00000100:00000010:0.0:1788574667.648938:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9aba80. 00000020:00000001:0.0:1788574668.097006:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574668.097010:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574668.097014:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574669.440915:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574669.440922:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574669.440926:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574669.440932:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574669.440936:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574669.440940:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574669.440976:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574669.440980:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574669.440983:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574670.145071:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574670.145079:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574670.145084:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574670.464903:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574670.464916:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574670.464920:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574670.464930:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574670.464940:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574670.464959:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574670.464964:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574670.464969:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574670.464977:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574671.488892:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574671.488899:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574671.488903:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574671.488911:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574671.488917:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574671.488920:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574671.488934:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574671.488948:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574671.488952:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574671.488978:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574671.488992:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574671.488994:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574671.488997:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574671.489000:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574671.489003:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574671.489005:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574671.489007:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574671.489008:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574672.193010:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574672.193014:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574672.193016:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574672.513021:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574672.513026:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574672.513028:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574672.513031:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574672.513034:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574672.513036:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574672.513039:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574672.513041:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574672.513044:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574672.768893:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310032621, at 4310032640 00000100:00000010:0.0:1788574672.768908:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab9c0. 00000020:00000001:0.0:1788574673.217000:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574673.217003:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574673.217006:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574673.536960:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574673.536970:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574673.536989:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574673.537002:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574673.537011:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574673.537015:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574673.537024:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574673.537028:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574673.537036:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574674.049008:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574674.049021:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574674.049024:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574674.049028:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574674.049033:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574674.049041:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15361 00000400:00000200:0.0:1788574674.049052:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574674.049058:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574674.049062:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574674.049065:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049068:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574674.049070:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049072:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574674.049076:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574674.049078:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574674.049080:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574674.049083:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574674.049085:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574674.049086:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574674.049088:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049090:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574674.049091:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049093:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574674.049096:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574674.049100:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049133:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574674.049137:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574674.049141:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574674.049145:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.049148:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574674.049151:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574674.049161:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574674.049165:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574674.049170:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574674.049172:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.049175:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574674.049179:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574674.049186:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574674.049196:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574674.049205:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574674.049219:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516726912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574674.049231:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574674.049236:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574674.049241:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516726912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574674.049267:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574674.049271:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574674.049275:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574674.049277:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574674.049279:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574674.049282:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574674.049285:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15361 00000400:00000200:0.0:1788574674.049290:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574674.049295:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574674.049297:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574674.049301:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049312:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574674.049315:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049318:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574674.049321:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574674.049322:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574674.049323:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574674.049326:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574674.049328:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574674.049331:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574674.049337:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049340:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574674.049342:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049345:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574674.049350:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574674.049357:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049360:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574674.049362:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574674.049371:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574674.049385:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.049395:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574674.049402:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574674.049426:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.049433:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574674.049441:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574674.049454:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574674.049460:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574674.049467:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.049475:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574674.049478:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574674.049484:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574674.049494:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574674.049500:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574674.049519:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516727040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574674.049532:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574674.049535:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574674.049540:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516727040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574674.049552:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574674.049554:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574674.049561:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574674.049562:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574674.049567:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574674.049577:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574674.049585:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15361 00000400:00000200:0.0:1788574674.049592:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574674.049596:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574674.049603:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574674.049613:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049621:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574674.049629:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049638:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574674.049645:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574674.049649:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574674.049653:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574674.049656:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574674.049660:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574674.049663:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574674.049668:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049673:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574674.049679:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049691:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574674.049696:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574674.049700:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049713:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574674.049717:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574674.049723:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574674.049730:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.049736:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574674.049738:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574674.049749:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.049755:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574674.049759:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574674.049762:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574674.049768:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574674.049771:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.049774:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574674.049778:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574674.049782:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574674.049790:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574674.049802:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574674.049829:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516727168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574674.049836:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574674.049837:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574674.049841:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516727168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574674.049848:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574674.049850:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574674.049854:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574674.049855:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574674.049856:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574674.049866:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574674.049871:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15361 00000400:00000200:0.0:1788574674.049892:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574674.049905:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574674.049920:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574674.049927:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049934:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574674.049946:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049956:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574674.049961:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574674.049964:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574674.049966:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574674.049969:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574674.049975:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574674.049979:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574674.049988:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.049997:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574674.050000:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.050003:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574674.050005:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574674.050007:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574674.050011:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574674.050016:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574674.050019:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574674.050021:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.050024:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574674.050028:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574674.050031:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.050034:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574674.050042:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574674.050052:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574674.050059:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574674.050060:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.050067:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574674.050072:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574674.050074:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574674.050079:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574674.050085:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574674.050096:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516727296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574674.050139:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574674.050141:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574674.050153:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516727296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574674.050179:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574674.050186:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574674.050192:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574674.050197:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574674.050201:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574674.050211:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574674.050219:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15361 00000400:00000200:0.0:1788574674.050232:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574674.050237:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574674.050238:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574674.050241:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.050243:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574674.050246:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.050255:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574674.050259:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574674.050264:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574674.050274:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574674.050278:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574674.050281:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574674.050288:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574674.050293:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.050296:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574674.050300:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574674.050308:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574674.050318:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574674.050323:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574674.050327:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574674.050332:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574674.050337:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574674.050341:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.050344:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574674.050347:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574674.050349:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.050354:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574674.050359:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574674.050368:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574674.050378:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574674.050382:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574674.050390:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574674.050397:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574674.050402:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574674.050412:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574674.050419:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574674.050435:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516727424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574674.050451:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574674.050461:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574674.050474:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516727424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574674.050557:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574674.050571:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15371) 00000100:00000001:0.0:1788574674.560958:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574674.560968:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574674.560974:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574674.560988:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574674.561004:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574674.561010:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574674.561022:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574674.561027:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574674.561033:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574675.265174:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574675.265178:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574675.265181:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574675.584981:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574675.584993:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574675.584999:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574675.585010:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574675.585019:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574675.585027:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574675.585034:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574675.585039:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574675.585043:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574676.609209:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574676.609213:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574676.609215:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574676.609219:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574676.609221:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574676.609223:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574676.609226:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574676.609228:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574676.609230:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574676.610171:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574676.610177:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574676.610179:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574676.610185:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574676.610204:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574676.610208:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574676.610210:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574676.610212:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574676.610214:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574677.313142:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574677.313150:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574677.313154:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574677.632966:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574677.632973:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574677.632980:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574677.632987:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574677.632996:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574677.633003:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574677.633009:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574677.633014:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574677.633022:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574677.888978:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310037742, at 4310037760 00000100:00000010:0.0:1788574677.889010:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abf60. 00000020:00000001:0.0:1788574678.336995:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574678.336999:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574678.337001:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574678.657182:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574678.657189:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574678.657195:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574678.657201:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574678.657207:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574678.657210:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574678.657213:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574678.657216:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574678.657219:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.169291:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574679.169304:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574679.169311:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574679.169329:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516727552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.169363:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574679.169399:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574679.169408:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574679.169415:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574679.169422:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574679.169433:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516727552:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574679.169458:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574679.169466:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574679.169475:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574679.169481:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574679.169491:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574679.169497:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574679.169509:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574679.169519:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574679.169534:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574679.169554:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516727552, portal 25 00000100:00000001:0.0:1788574679.169562:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574679.169565:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788574679.169578:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516727552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.169590:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574679.169597:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574679.169601:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574679.169607:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516727552, offset 0 00000400:00000200:0.0:1788574679.169621:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574679.169639:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574679.169656:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574679.169668:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574679.169678:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574679.169688:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574679.169699:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574679.169711:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547336:547336:256:4294967295] 192.168.202.128@tcp LPNI seq info [547259:547259:8:4294967295] 00000400:00000200:0.0:1788574679.169729:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574679.169754:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574679.169781:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574679.169794:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574679.169842:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574679.169848:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574679.169880:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574679.169884:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574679.169891:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574679.169906:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574679.169922:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574679.169928:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574679.169939:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574679.169946:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516727552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.169967:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574679.169978:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516727552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.169998:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.170003:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574679.170014:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574679.170027:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574679.170035:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310042862 00000100:00000010:0.0:1788574679.170047:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574679.170059:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574679.170068:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310042862 00000100:00000010:0.0:1788574679.170074:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50600. 00000100:00000001:0.0:1788574679.170085:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574679.170094:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574679.170101:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.170108:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574679.170124:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574679/real 1788574679] req@ffff8fc4ac821c00 x1875440516727552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574679.170188:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574679.170193:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574679.170197:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574679.170203:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574679.170209:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574679.170213:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574679.170224:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516727552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574679.170267:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516727552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.170299:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574679.170304:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.170311:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574679.170315:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574679.170325:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516727552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.170348:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574679.170361:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516727552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.170377:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574679.170384:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.170389:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574679.170393:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.170397:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574679.170402:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574679.170407:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574679.170414:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574679.170425:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574679.170452:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516727552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574679.170502:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516727552:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574679.170515:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.170520:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574679.170528:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516727552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.170561:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574679.170567:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574679.170572:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574679.170579:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574679.170586:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574679.170591:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574679.170598:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574679.170604:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788574679.170612:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574679.170616:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574679.170622:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788574679.170629:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574679.170634:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.170639:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.170651:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574679.170656:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.170663:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574679.170668:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574679.170672:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574679.170682:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516727680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.170714:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574679.170719:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574679.170725:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574679.170730:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574679.170740:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574679.170750:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516727680:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574679.170768:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574679.170774:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574679.170778:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574679.170784:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574679.170790:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574679.170796:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574679.170832:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574679.170842:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574679.170853:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574679.170874:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516727680, portal 10 00000100:00000001:0.0:1788574679.170882:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574679.170886:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788574679.170899:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516727680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.170935:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574679.170941:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574679.170947:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574679.170957:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516727680, offset 0 00000400:00000200:0.0:1788574679.170972:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574679.170990:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574679.171003:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574679.171009:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574679.171015:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574679.171020:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574679.171025:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574679.171029:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547337:547337:256:4294967295] 192.168.202.128@tcp LPNI seq info [547260:547260:8:4294967295] 00000400:00000200:0.0:1788574679.171035:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574679.171044:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574679.171050:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574679.171054:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574679.171058:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574679.171068:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574679.171081:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574679.171083:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574679.171087:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574679.171091:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574679.171095:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574679.171097:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574679.171100:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574679.171105:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516727680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171115:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574679.171118:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516727680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171125:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.171127:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574679.171131:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574679.171135:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574679.171139:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574679.171141:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310039060 00000100:00000001:0.0:1788574679.171144:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310042862 00000100:00000001:0.0:1788574679.171146:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310039060 00000100:00000001:0.0:1788574679.171148:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310042862 00000100:00000001:0.0:1788574679.171151:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574679.171153:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.171155:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574679.171160:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574679/real 1788574679] req@ffff8fc4ac820a80 x1875440516727680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574679.171168:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574679.171170:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574679.171176:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574679.171179:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574679.171180:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574679.171183:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574679.171187:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516727680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574679.171196:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516727680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171204:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574679.171205:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.171207:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574679.171209:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574679.171213:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516727680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171220:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574679.171225:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516727680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171232:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574679.171235:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.171237:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574679.171238:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.171240:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574679.171242:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574679.171245:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574679.171248:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574679.171251:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574679.171255:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516727680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574679.171264:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516727680:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574679.171274:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574679.171279:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516727808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171287:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574679.171289:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574679.171292:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574679.171294:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574679.171296:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574679.171299:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516727808:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574679.171302:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574679.171305:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574679.171306:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574679.171308:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574679.171312:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574679.171314:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574679.171317:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574679.171320:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574679.171324:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574679.171327:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516727808, portal 10 00000100:00000001:0.0:1788574679.171329:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574679.171330:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788574679.171336:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516727808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171343:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574679.171346:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574679.171348:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574679.171351:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516727808, offset 0 00000400:00000200:0.0:1788574679.171355:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574679.171358:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574679.171362:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574679.171364:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574679.171372:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574679.171374:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574679.171377:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574679.171380:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547338:547338:256:4294967295] 192.168.202.128@tcp LPNI seq info [547261:547261:8:4294967295] 00000400:00000200:0.0:1788574679.171385:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574679.171392:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574679.171397:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574679.171400:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574679.171404:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574679.171407:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574679.171420:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574679.171422:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574679.171425:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574679.171428:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574679.171431:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574679.171433:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574679.171436:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574679.171440:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516727808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171448:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574679.171452:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516727808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171459:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.171461:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574679.171464:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574679.171467:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574679.171470:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574679.171472:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310039067 00000100:00000001:0.0:1788574679.171475:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310042862 00000100:00000001:0.0:1788574679.171477:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310039067 00000100:00000001:0.0:1788574679.171483:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310042862 00000100:00000001:0.0:1788574679.171485:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574679.171488:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.171490:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574679.171494:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574679/real 1788574679] req@ffff8fc4ac823800 x1875440516727808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574679.171503:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574679.171505:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574679.171507:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574679.171509:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574679.171511:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574679.171513:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574679.171516:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516727808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574679.171526:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516727808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171533:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574679.171534:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.171536:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574679.171537:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574679.171541:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516727808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171550:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574679.171553:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516727808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171561:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574679.171564:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.171565:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574679.171567:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.171572:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574679.171575:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574679.171578:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574679.171582:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574679.171584:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574679.171589:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516727808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574679.171597:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516727808:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574679.171601:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574679.171605:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516727936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171614:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574679.171616:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574679.171618:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574679.171620:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574679.171622:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574679.171625:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516727936:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574679.171629:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574679.171631:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574679.171633:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574679.171635:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574679.171637:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574679.171639:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574679.171643:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574679.171646:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574679.171649:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574679.171652:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516727936, portal 4 00000100:00000001:0.0:1788574679.171654:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574679.171656:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788574679.171662:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516727936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171672:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574679.171675:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574679.171677:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574679.171680:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516727936, offset 0 00000400:00000200:0.0:1788574679.171684:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574679.171689:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574679.171692:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574679.171695:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574679.171698:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574679.171700:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574679.171703:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574679.171706:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547339:547339:256:4294967295] 192.168.202.128@tcp LPNI seq info [547262:547262:8:4294967295] 00000400:00000200:0.0:1788574679.171711:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574679.171718:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574679.171723:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574679.171726:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574679.171731:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574679.171734:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574679.171746:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574679.171748:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574679.171750:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574679.171754:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574679.171757:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574679.171759:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574679.171762:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574679.171765:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516727936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171774:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574679.171777:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516727936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171795:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.171797:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574679.171800:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574679.171804:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574679.171827:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574679.171830:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310039208 00000100:00000001:0.0:1788574679.171832:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310042862 00000100:00000001:0.0:1788574679.171835:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310039208 00000100:00000001:0.0:1788574679.171837:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310042862 00000100:00000001:0.0:1788574679.171839:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574679.171841:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.171843:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574679.171848:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574679/real 1788574679] req@ffff8fc4ac820e00 x1875440516727936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574679.171856:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574679.171876:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574679.171879:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574679.171882:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574679.171884:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574679.171886:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574679.171890:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516727936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574679.171899:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516727936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171906:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574679.171908:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.171910:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574679.171912:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574679.171915:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516727936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171928:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574679.171933:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516727936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171940:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574679.171943:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.171944:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574679.171945:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.171947:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574679.171950:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574679.171953:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574679.171957:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574679.171959:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574679.171964:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516727936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574679.171971:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516727936:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574679.171976:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.171978:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574679.171981:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516727680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.171989:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574679.171991:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574679.171993:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574679.171996:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574679.171998:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574679.172000:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574679.172002:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574679.172003:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788574679.172012:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574679.172013:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574679.172015:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788574679.172018:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574679.172019:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.172021:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574679.172025:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516727808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.172032:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574679.172033:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574679.172035:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574679.172038:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574679.172040:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574679.172041:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574679.172042:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574679.172044:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788574679.172046:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574679.172047:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574679.172049:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788574679.172051:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574679.172053:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.172054:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574679.172058:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516727936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.172064:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574679.172065:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574679.172067:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574679.172069:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574679.172070:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574679.172072:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574679.172074:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574679.172075:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788574679.172078:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574679.172079:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574679.172081:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788574679.172083:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574679.172088:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.172090:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.172094:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574679.172095:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.172098:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574679.172100:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574679.172102:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574679.172107:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516728064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.172114:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574679.172116:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574679.172118:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574679.172120:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574679.172122:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574679.172125:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516728064:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574679.172129:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574679.172131:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574679.172133:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574679.172135:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574679.172137:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574679.172140:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574679.172144:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574679.172147:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574679.172150:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574679.172153:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516728064, portal 4 00000100:00000001:0.0:1788574679.172155:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574679.172157:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788574679.172162:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516728064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.172170:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574679.172173:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574679.172175:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574679.172177:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516728064, offset 0 00000400:00000200:0.0:1788574679.172181:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574679.172189:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574679.172193:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574679.172196:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574679.172199:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574679.172202:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574679.172204:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574679.172208:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547340:547340:256:4294967295] 192.168.202.128@tcp LPNI seq info [547263:547263:8:4294967295] 00000400:00000200:0.0:1788574679.172213:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574679.172220:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574679.172224:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574679.172228:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574679.172232:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574679.172235:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574679.172247:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574679.172249:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574679.172252:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574679.172256:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574679.172259:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574679.172261:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574679.172263:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574679.172267:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516728064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.172275:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574679.172279:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516728064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574704 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.172286:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.172287:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574679.172290:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574679.172293:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574679.172297:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574679.172303:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310039210 00000100:00000001:0.0:1788574679.172306:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310042862 00000100:00000001:0.0:1788574679.172308:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310039210 00000100:00000001:0.0:1788574679.172310:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310042862 00000100:00000001:0.0:1788574679.172313:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574679.172315:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.172317:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574679.172321:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574679/real 1788574679] req@ffff8fc4ac821880 x1875440516728064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574679.172330:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574679.172331:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574679.172333:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574679.172335:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574679.172337:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574679.172340:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574679.172343:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516728064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574679.172352:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516728064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.172359:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574679.172361:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.172363:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574679.172364:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574679.172368:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516728064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.172376:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574679.172380:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516728064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.172387:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574679.172393:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.172395:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574679.172396:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.172398:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574679.172400:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574679.172403:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574679.172407:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574679.172409:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574679.172413:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516728064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574679.172422:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516728064:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574679.172426:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.172428:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574679.172431:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516728064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574704 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574679.172439:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574679.172440:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574679.172442:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574679.172445:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574679.172446:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574679.172448:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574679.172450:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574679.172452:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788574679.172455:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574679.172456:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574679.172458:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788574679.172460:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574679.172461:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.172463:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574679.172466:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574679.172468:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574679.172470:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574679.172475:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574680.385064:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574680.385069:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574680.385073:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574680.704984:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574680.704995:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574680.705002:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574680.705008:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574680.705014:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574680.705017:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574680.705031:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574680.705035:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574680.705040:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574681.728974:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574681.728982:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574681.728990:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574681.728998:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574681.729008:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574681.729015:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574681.729021:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574681.729026:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574681.729033:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574682.432998:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574682.433002:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574682.433005:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574682.752964:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574682.752978:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574682.752993:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574682.753010:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574682.753017:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574682.753025:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574682.753057:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574682.753060:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574682.753064:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574683.008939:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310042862, at 4310042880 00000100:00000010:0.0:1788574683.008955:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574683.457056:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574683.457066:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574683.457068:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574683.776946:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574683.776951:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574683.776954:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574683.776958:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574683.776962:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574683.776967:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574683.776970:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574683.776973:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574683.776975:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.289210:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574684.289215:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574684.289218:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574684.289227:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6680 x1875440516728192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.289242:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574684.289246:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574684.289253:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574684.289259:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574684.289266:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574684.289276:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516728192:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574684.289283:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574684.289290:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574684.289295:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574684.289301:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574684.289310:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574684.289315:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574684.289326:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574684.289333:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574684.289345:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574684.289357:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516728192, portal 25 00000100:00000001:0.0:1788574684.289362:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574684.289367:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605330560 : -123400104221056 : ffff8fc4ac9f6680) 00000100:00000040:0.0:1788574684.289378:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6680 x1875440516728192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.289408:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574684.289418:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574684.289424:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574684.289432:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516728192, offset 0 00000400:00000200:0.0:1788574684.289445:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574684.289457:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574684.289467:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574684.289475:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574684.289481:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574684.289488:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574684.289496:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574684.289502:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547341:547341:256:4294967295] 192.168.202.128@tcp LPNI seq info [547264:547264:8:4294967295] 00000400:00000200:0.0:1788574684.289515:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574684.289535:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574684.289547:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574684.289556:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574684.289570:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574684.289578:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788574684.289609:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574684.289614:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574684.289624:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574684.289635:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574684.289646:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574684.289652:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574684.289660:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574684.289669:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6680 x1875440516728192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.289692:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574684.289700:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6680 x1875440516728192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.289732:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.289736:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574684.289746:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574684.289770:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574684.289773:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310047981 00000100:00000010:0.0:1788574684.289778:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574684.289789:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574684.289799:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310047981 00000100:00000010:0.0:1788574684.289805:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50060. 00000100:00000001:0.0:1788574684.289834:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574684.289839:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574684.289846:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.289853:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574684.289864:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574684/real 1788574684] req@ffff8fc4ac9f6680 x1875440516728192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574684.289891:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574684.289895:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574684.289899:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574684.289904:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574684.289910:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574684.289915:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574684.289922:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6680 x1875440516728192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574684.289939:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6680 x1875440516728192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.289952:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574684.289956:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.289963:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574684.289968:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574684.289975:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6680 x1875440516728192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290003:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574684.290014:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6680 x1875440516728192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290042:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574684.290049:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290051:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574684.290053:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290056:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574684.290059:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574684.290063:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574684.290068:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574684.290071:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574684.290076:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6680 x1875440516728192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574684.290085:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516728192:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574684.290091:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290093:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574684.290096:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6680 x1875440516728192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290103:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574684.290106:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574684.290109:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574684.290113:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574684.290115:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574684.290117:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574684.290119:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574684.290122:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788574684.290126:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574684.290133:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574684.290135:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6680. 00000100:00000001:0.0:1788574684.290139:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574684.290140:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290143:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290152:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574684.290154:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.290156:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574684.290157:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574684.290159:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574684.290163:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7480 x1875440516728320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290170:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574684.290171:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574684.290173:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574684.290175:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574684.290177:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574684.290180:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516728320:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574684.290183:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574684.290185:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574684.290187:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574684.290188:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574684.290192:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574684.290193:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574684.290198:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574684.290201:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574684.290205:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574684.290208:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516728320, portal 10 00000100:00000001:0.0:1788574684.290210:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574684.290211:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605334144 : -123400104217472 : ffff8fc4ac9f7480) 00000100:00000040:0.0:1788574684.290219:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7480 x1875440516728320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290225:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574684.290228:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574684.290235:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574684.290238:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516728320, offset 0 00000400:00000200:0.0:1788574684.290242:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574684.290247:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574684.290250:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574684.290253:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574684.290256:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574684.290258:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574684.290261:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574684.290264:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547342:547342:256:4294967295] 192.168.202.128@tcp LPNI seq info [547265:547265:8:4294967295] 00000400:00000200:0.0:1788574684.290268:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574684.290273:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574684.290278:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574684.290281:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574684.290285:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574684.290288:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788574684.290299:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574684.290301:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574684.290304:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574684.290307:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574684.290310:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574684.290312:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574684.290315:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574684.290318:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7480 x1875440516728320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290323:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574684.290325:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7480 x1875440516728320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290329:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.290330:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574684.290337:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574684.290340:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574684.290343:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574684.290344:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310044179 00000100:00000001:0.0:1788574684.290346:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310047981 00000100:00000001:0.0:1788574684.290348:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310044179 00000100:00000001:0.0:1788574684.290349:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310047981 00000100:00000001:0.0:1788574684.290351:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574684.290352:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.290353:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574684.290356:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574684/real 1788574684] req@ffff8fc4ac9f7480 x1875440516728320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574684.290360:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574684.290361:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574684.290363:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574684.290364:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574684.290365:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574684.290367:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574684.290368:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7480 x1875440516728320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574684.290373:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7480 x1875440516728320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290376:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574684.290377:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290378:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574684.290379:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574684.290381:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7480 x1875440516728320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290384:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574684.290387:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7480 x1875440516728320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290393:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574684.290395:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290396:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574684.290397:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290398:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574684.290400:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574684.290401:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574684.290403:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574684.290404:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574684.290406:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7480 x1875440516728320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574684.290411:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516728320:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574684.290414:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574684.290415:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4e00 x1875440516728448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290419:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574684.290419:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574684.290421:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574684.290422:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574684.290423:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574684.290424:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516728448:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574684.290426:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574684.290427:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574684.290428:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574684.290429:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574684.290431:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574684.290432:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574684.290438:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574684.290439:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574684.290442:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574684.290444:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516728448, portal 10 00000100:00000001:0.0:1788574684.290445:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574684.290446:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605324288 : -123400104227328 : ffff8fc4ac9f4e00) 00000100:00000040:0.0:1788574684.290450:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4e00 x1875440516728448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290454:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574684.290456:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574684.290457:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574684.290458:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516728448, offset 0 00000400:00000200:0.0:1788574684.290461:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574684.290463:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574684.290465:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574684.290467:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574684.290468:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574684.290470:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574684.290472:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574684.290474:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547343:547343:256:4294967295] 192.168.202.128@tcp LPNI seq info [547266:547266:8:4294967295] 00000400:00000200:0.0:1788574684.290476:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574684.290480:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574684.290483:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574684.290485:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574684.290487:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574684.290489:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788574684.290499:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574684.290501:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574684.290504:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574684.290507:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574684.290511:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574684.290516:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574684.290519:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574684.290523:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4e00 x1875440516728448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290531:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574684.290534:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4e00 x1875440516728448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290540:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.290542:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574684.290545:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574684.290548:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574684.290552:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574684.290553:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310044186 00000100:00000001:0.0:1788574684.290556:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310047981 00000100:00000001:0.0:1788574684.290558:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310044186 00000100:00000001:0.0:1788574684.290561:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310047981 00000100:00000001:0.0:1788574684.290563:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574684.290565:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.290567:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574684.290571:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574684/real 1788574684] req@ffff8fc4ac9f4e00 x1875440516728448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574684.290579:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574684.290580:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574684.290582:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574684.290584:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574684.290586:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574684.290588:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574684.290592:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4e00 x1875440516728448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574684.290599:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4e00 x1875440516728448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290611:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574684.290613:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290615:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574684.290616:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574684.290620:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4e00 x1875440516728448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290627:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574684.290631:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4e00 x1875440516728448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290637:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574684.290639:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290641:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574684.290642:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290644:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574684.290647:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574684.290650:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574684.290653:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574684.290655:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574684.290659:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4e00 x1875440516728448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574684.290664:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516728448:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574684.290666:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574684.290668:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516728576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290673:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574684.290674:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574684.290679:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574684.290681:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574684.290682:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574684.290684:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516728576:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574684.290686:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574684.290687:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574684.290689:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574684.290690:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574684.290692:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574684.290693:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574684.290696:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574684.290698:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574684.290700:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574684.290702:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516728576, portal 4 00000100:00000001:0.0:1788574684.290703:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574684.290704:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788574684.290708:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516728576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290713:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574684.290715:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574684.290717:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574684.290719:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516728576, offset 0 00000400:00000200:0.0:1788574684.290723:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574684.290727:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574684.290730:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574684.290733:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574684.290735:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574684.290738:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574684.290740:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574684.290743:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547344:547344:256:4294967295] 192.168.202.128@tcp LPNI seq info [547267:547267:8:4294967295] 00000400:00000200:0.0:1788574684.290747:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574684.290764:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574684.290774:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574684.290778:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574684.290782:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574684.290784:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788574684.290794:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574684.290796:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574684.290799:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574684.290802:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574684.290806:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574684.290827:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574684.290831:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574684.290835:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516728576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290842:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574684.290846:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516728576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290852:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.290853:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574684.290857:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574684.290860:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574684.290863:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574684.290865:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310044327 00000100:00000001:0.0:1788574684.290868:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310047981 00000100:00000001:0.0:1788574684.290870:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310044327 00000100:00000001:0.0:1788574684.290872:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310047981 00000100:00000001:0.0:1788574684.290874:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574684.290877:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.290879:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574684.290882:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574684/real 1788574684] req@ffff8fc4ac9f6a00 x1875440516728576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574684.290896:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574684.290897:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574684.290899:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574684.290901:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574684.290903:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574684.290906:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574684.290909:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516728576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574684.290917:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516728576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290924:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574684.290925:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290927:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574684.290928:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574684.290932:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516728576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290938:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574684.290941:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516728576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.290948:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574684.290950:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290952:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574684.290953:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.290955:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574684.290957:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574684.290959:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574684.290963:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574684.290965:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574684.290979:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516728576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574684.290987:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516728576:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574684.290991:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574684.290994:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5f80 x1875440516728704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.291000:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574684.291002:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574684.291004:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574684.291006:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574684.291008:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574684.291010:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516728704:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574684.291014:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574684.291015:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574684.291017:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574684.291019:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574684.291023:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574684.291025:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574684.291028:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574684.291031:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574684.291034:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574684.291037:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516728704, portal 4 00000100:00000001:0.0:1788574684.291039:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574684.291040:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605328768 : -123400104222848 : ffff8fc4ac9f5f80) 00000100:00000040:0.0:1788574684.291046:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5f80 x1875440516728704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.291053:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574684.291055:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574684.291057:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574684.291060:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516728704, offset 0 00000400:00000200:0.0:1788574684.291069:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574684.291073:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574684.291076:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574684.291079:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574684.291081:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574684.291084:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574684.291086:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574684.291089:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547345:547345:256:4294967295] 192.168.202.128@tcp LPNI seq info [547268:547268:8:4294967295] 00000400:00000200:0.0:1788574684.291093:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574684.291099:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574684.291103:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574684.291106:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574684.291110:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574684.291113:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788574684.291123:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574684.291124:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574684.291127:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574684.291130:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574684.291134:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574684.291135:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574684.291138:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574684.291142:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5f80 x1875440516728704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.291149:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574684.291152:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5f80 x1875440516728704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574709 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.291159:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.291160:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574684.291163:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574684.291166:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574684.291169:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574684.291177:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310044329 00000100:00000001:0.0:1788574684.291180:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310047981 00000100:00000001:0.0:1788574684.291182:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310044329 00000100:00000001:0.0:1788574684.291184:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310047981 00000100:00000001:0.0:1788574684.291186:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574684.291188:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.291190:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574684.291194:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574684/real 1788574684] req@ffff8fc4ac9f5f80 x1875440516728704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574684.291203:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574684.291204:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574684.291206:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574684.291208:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574684.291210:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574684.291213:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574684.291216:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5f80 x1875440516728704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574684.291224:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5f80 x1875440516728704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.291231:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574684.291232:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.291234:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574684.291235:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574684.291239:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5f80 x1875440516728704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.291246:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574684.291250:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5f80 x1875440516728704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.291256:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574684.291264:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.291265:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574684.291266:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.291268:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574684.291270:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574684.291273:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574684.291277:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574684.291279:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574684.291283:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5f80 x1875440516728704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574684.291291:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516728704:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574684.291295:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.291297:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574684.291300:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7480 x1875440516728320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.291306:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574684.291308:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574684.291310:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574684.291312:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574684.291314:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574684.291315:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574684.291317:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574684.291319:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35800. 02000000:00000001:0.0:1788574684.291322:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574684.291323:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574684.291324:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7480. 00000100:00000001:0.0:1788574684.291327:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574684.291329:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.291330:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574684.291333:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4e00 x1875440516728448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.291351:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574684.291352:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574684.291354:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574684.291357:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574684.291358:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574684.291359:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574684.291361:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574684.291363:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788574684.291365:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574684.291366:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574684.291367:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4e00. 00000100:00000001:0.0:1788574684.291370:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574684.291371:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.291373:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574684.291377:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516728576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.291383:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574684.291384:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574684.291386:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574684.291388:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574684.291389:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574684.291390:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574684.291392:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574684.291393:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788574684.291395:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574684.291397:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574684.291398:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788574684.291400:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574684.291401:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.291403:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574684.291406:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5f80 x1875440516728704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574709 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574684.291413:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574684.291414:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574684.291421:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574684.291423:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574684.291425:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574684.291426:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574684.291428:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574684.291430:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788574684.291432:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574684.291433:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574684.291435:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5f80. 00000100:00000001:0.0:1788574684.291437:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574684.291439:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.291441:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.291444:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574684.291446:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.291447:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574684.291449:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.800942:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574684.800951:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574684.800957:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574684.800964:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574684.800971:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574684.800976:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.801003:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574684.801008:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.801013:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574684.801042:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574684.801052:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574684.801060:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574684.801067:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574684.801076:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574684.801083:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.801088:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574684.801090:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574684.801094:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574685.504986:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574685.504989:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574685.504992:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574685.824942:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574685.824966:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574685.824970:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574685.824982:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574685.824998:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574685.825004:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574685.825039:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574685.825045:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574685.825053:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574686.848900:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574686.848907:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574686.848909:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574686.848915:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574686.848921:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574686.848926:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574686.848935:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574686.848939:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574686.848943:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574686.849007:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574686.849015:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574686.849021:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574686.849026:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574686.849030:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574686.849034:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574686.849038:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574686.849040:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574686.849042:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574687.360923:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788574687.360947:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788574687.360954:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788574687.361119:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574687.361135:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574687.361140:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574687.361142:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788574687.552967:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574687.552971:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574687.552974:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574687.872878:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574687.872899:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574687.872903:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574687.872907:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574687.872912:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574687.872918:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574687.872943:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574687.872947:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574687.872950:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574687.872975:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574687.872979:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574687.872982:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574687.872985:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574687.872992:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574687.872996:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574687.873002:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574687.873005:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574687.873008:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574688.128889:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310047981, at 4310048000 00000100:00000010:0.0:1788574688.128907:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574688.576984:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574688.576988:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574688.576992:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574688.896948:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574688.896956:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574688.896959:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574688.896966:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574688.896974:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574688.896983:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574688.896993:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574688.897001:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574688.897011:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.409207:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574689.409219:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574689.409227:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574689.409250:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516728832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.409303:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574689.409327:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574689.409339:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574689.409345:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574689.409356:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574689.409374:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516728832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574689.409384:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574689.409393:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574689.409398:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574689.409404:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574689.409415:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574689.409428:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574689.409446:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574689.409458:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574689.409476:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574689.409491:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516728832, portal 25 00000100:00000001:0.0:1788574689.409504:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574689.409512:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788574689.409535:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516728832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.409570:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574689.409578:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574689.409585:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574689.409594:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516728832, offset 0 00000400:00000200:0.0:1788574689.409612:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574689.409626:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574689.409641:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574689.409646:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574689.409655:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574689.409663:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574689.409673:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574689.409692:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547346:547346:256:4294967295] 192.168.202.128@tcp LPNI seq info [547269:547269:8:4294967295] 00000400:00000200:0.0:1788574689.409706:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574689.409746:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574689.409774:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574689.409784:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574689.409800:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574689.409841:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574689.409895:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574689.409903:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574689.409919:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574689.409944:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574689.409960:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574689.409967:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574689.409979:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574689.409994:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516728832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410022:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574689.410037:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516728832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410077:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.410085:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574689.410097:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574689.410115:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574689.410125:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310053102 00000100:00000010:0.0:1788574689.410137:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574689.410150:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574689.410167:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310053102 00000100:00000010:0.0:1788574689.410175:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50360. 00000100:00000001:0.0:1788574689.410186:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574689.410197:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574689.410205:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.410213:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574689.410230:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574689/real 1788574689] req@ffff8fc4901a4a80 x1875440516728832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574689.410345:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574689.410351:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574689.410354:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574689.410358:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574689.410361:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574689.410366:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574689.410376:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516728832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574689.410392:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516728832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410403:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574689.410408:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.410412:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574689.410415:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574689.410421:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516728832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410429:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574689.410435:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516728832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410444:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574689.410448:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.410450:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574689.410451:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.410454:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574689.410456:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574689.410459:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574689.410464:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574689.410467:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574689.410471:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516728832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574689.410487:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516728832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574689.410492:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.410494:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574689.410497:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516728832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410503:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574689.410507:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574689.410510:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574689.410514:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574689.410516:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574689.410518:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574689.410520:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574689.410523:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788574689.410526:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574689.410528:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574689.410530:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788574689.410534:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574689.410535:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.410538:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.410544:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574689.410546:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.410549:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574689.410551:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574689.410553:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574689.410557:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516728960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410564:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574689.410566:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574689.410569:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574689.410571:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574689.410574:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574689.410577:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516728960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574689.410585:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574689.410588:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574689.410590:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574689.410591:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574689.410594:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574689.410596:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574689.410601:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574689.410604:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574689.410608:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574689.410611:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516728960, portal 10 00000100:00000001:0.0:1788574689.410613:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574689.410614:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788574689.410621:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516728960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410628:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574689.410631:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574689.410633:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574689.410636:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516728960, offset 0 00000400:00000200:0.0:1788574689.410640:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574689.410645:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574689.410649:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574689.410652:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574689.410655:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574689.410658:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574689.410661:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574689.410664:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547347:547347:256:4294967295] 192.168.202.128@tcp LPNI seq info [547270:547270:8:4294967295] 00000400:00000200:0.0:1788574689.410668:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574689.410675:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574689.410680:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574689.410683:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574689.410687:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574689.410695:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574689.410709:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574689.410711:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574689.410714:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574689.410718:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574689.410722:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574689.410724:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574689.410727:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574689.410731:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516728960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410739:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574689.410742:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516728960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410749:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.410750:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574689.410754:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574689.410757:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574689.410761:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574689.410764:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310049299 00000100:00000001:0.0:1788574689.410767:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310053102 00000100:00000001:0.0:1788574689.410770:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310049299 00000100:00000001:0.0:1788574689.410772:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310053102 00000100:00000001:0.0:1788574689.410774:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574689.410776:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.410778:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574689.410782:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574689/real 1788574689] req@ffff8fc4901a7100 x1875440516728960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574689.410790:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574689.410791:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574689.410800:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574689.410803:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574689.410805:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574689.410831:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574689.410836:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516728960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574689.410844:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516728960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410850:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574689.410851:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.410854:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574689.410855:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574689.410859:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516728960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410865:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574689.410869:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516728960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410876:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574689.410878:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.410879:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574689.410881:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.410882:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574689.410885:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574689.410887:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574689.410892:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574689.410894:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574689.410897:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516728960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574689.410905:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516728960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574689.410914:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574689.410919:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516729088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410926:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574689.410928:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574689.410930:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574689.410932:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574689.410934:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574689.410937:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516729088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574689.410940:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574689.410943:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574689.410945:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574689.410946:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574689.410949:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574689.410951:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574689.410955:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574689.410957:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574689.410961:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574689.410964:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516729088, portal 10 00000100:00000001:0.0:1788574689.410966:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574689.410968:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788574689.410974:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516729088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.410980:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574689.410983:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574689.410986:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574689.410988:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516729088, offset 0 00000400:00000200:0.0:1788574689.410992:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574689.410996:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574689.410999:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574689.411003:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574689.411010:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574689.411013:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574689.411016:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574689.411020:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547348:547348:256:4294967295] 192.168.202.128@tcp LPNI seq info [547271:547271:8:4294967295] 00000400:00000200:0.0:1788574689.411024:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574689.411031:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574689.411036:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574689.411040:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574689.411044:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574689.411047:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574689.411058:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574689.411060:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574689.411062:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574689.411066:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574689.411069:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574689.411071:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574689.411074:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574689.411078:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516729088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411086:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574689.411089:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516729088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411096:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.411098:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574689.411101:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574689.411104:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574689.411108:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574689.411110:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310049307 00000100:00000001:0.0:1788574689.411113:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310053102 00000100:00000001:0.0:1788574689.411115:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310049307 00000100:00000001:0.0:1788574689.411122:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310053102 00000100:00000001:0.0:1788574689.411125:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574689.411127:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.411129:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574689.411134:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574689/real 1788574689] req@ffff8fc4901a5180 x1875440516729088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574689.411142:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574689.411144:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574689.411146:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574689.411148:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574689.411150:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574689.411152:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574689.411156:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516729088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574689.411164:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516729088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411170:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574689.411172:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.411174:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574689.411175:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574689.411179:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516729088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411186:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574689.411190:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516729088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411197:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574689.411200:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.411201:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574689.411202:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.411210:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574689.411213:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574689.411215:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574689.411220:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574689.411222:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574689.411226:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516729088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574689.411233:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516729088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574689.411237:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574689.411241:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516729216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411248:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574689.411249:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574689.411252:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574689.411254:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574689.411256:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574689.411259:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516729216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574689.411262:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574689.411265:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574689.411266:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574689.411268:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574689.411271:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574689.411273:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574689.411277:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574689.411279:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574689.411283:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574689.411286:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516729216, portal 4 00000100:00000001:0.0:1788574689.411302:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574689.411303:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788574689.411309:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516729216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411321:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574689.411325:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574689.411327:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574689.411330:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516729216, offset 0 00000400:00000200:0.0:1788574689.411334:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574689.411339:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574689.411343:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574689.411345:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574689.411348:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574689.411350:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574689.411353:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574689.411355:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547349:547349:256:4294967295] 192.168.202.128@tcp LPNI seq info [547272:547272:8:4294967295] 00000400:00000200:0.0:1788574689.411360:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574689.411366:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574689.411370:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574689.411374:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574689.411379:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574689.411381:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574689.411392:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574689.411394:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574689.411397:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574689.411400:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574689.411403:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574689.411405:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574689.411408:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574689.411412:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516729216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411420:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574689.411423:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516729216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411441:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.411443:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574689.411447:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574689.411450:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574689.411453:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574689.411456:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310049447 00000100:00000001:0.0:1788574689.411458:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310053102 00000100:00000001:0.0:1788574689.411460:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310049447 00000100:00000001:0.0:1788574689.411462:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310053102 00000100:00000001:0.0:1788574689.411464:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574689.411466:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.411469:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574689.411473:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574689/real 1788574689] req@ffff8fc4901a5c00 x1875440516729216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574689.411480:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574689.411482:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574689.411483:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574689.411486:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574689.411488:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574689.411490:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574689.411493:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516729216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574689.411501:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516729216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411507:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574689.411509:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.411510:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574689.411512:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574689.411516:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516729216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411529:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574689.411533:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516729216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411540:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574689.411543:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.411545:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574689.411546:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.411548:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574689.411550:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574689.411552:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574689.411556:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574689.411558:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574689.411561:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516729216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574689.411570:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516729216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574689.411574:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574689.411578:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516729344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411585:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574689.411587:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574689.411589:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574689.411591:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574689.411593:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574689.411596:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516729344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574689.411600:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574689.411602:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574689.411607:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574689.411609:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574689.411612:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574689.411614:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574689.411618:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574689.411621:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574689.411624:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574689.411627:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516729344, portal 4 00000100:00000001:0.0:1788574689.411629:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574689.411630:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788574689.411636:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516729344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411643:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574689.411645:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574689.411648:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574689.411650:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516729344, offset 0 00000400:00000200:0.0:1788574689.411654:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574689.411658:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574689.411662:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574689.411665:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574689.411667:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574689.411669:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574689.411671:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574689.411674:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547350:547350:256:4294967295] 192.168.202.128@tcp LPNI seq info [547273:547273:8:4294967295] 00000400:00000200:0.0:1788574689.411678:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574689.411684:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574689.411689:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574689.411692:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574689.411696:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574689.411698:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574689.411710:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574689.411716:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574689.411719:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574689.411723:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574689.411726:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574689.411728:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574689.411731:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574689.411734:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516729344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411764:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574689.411768:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516729344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574714 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411775:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.411778:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574689.411782:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574689.411786:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574689.411790:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574689.411792:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310049449 00000100:00000001:0.0:1788574689.411797:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310053102 00000100:00000001:0.0:1788574689.411799:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310049449 00000100:00000001:0.0:1788574689.411802:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310053102 00000100:00000001:0.0:1788574689.411804:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574689.411806:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.411843:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574689.411847:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574689/real 1788574689] req@ffff8fc4901a7800 x1875440516729344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574689.411855:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574689.411857:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574689.411860:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574689.411867:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574689.411869:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574689.411880:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574689.411885:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516729344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574689.411893:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516729344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411899:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574689.411901:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.411903:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574689.411904:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574689.411908:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516729344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411914:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574689.411918:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516729344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411925:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574689.411928:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.411929:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574689.411931:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.411933:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574689.411935:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574689.411938:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574689.411942:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574689.411944:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574689.411948:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516729344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574689.411957:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516729344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574689.411961:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.411963:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574689.411971:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516728960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.411978:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574689.411980:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574689.411982:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574689.411985:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574689.411986:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574689.411988:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574689.411990:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574689.411992:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788574689.411995:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574689.411997:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574689.411998:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788574689.412002:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574689.412003:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.412005:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574689.412009:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516729088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.412016:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574689.412017:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574689.412019:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574689.412021:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574689.412022:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574689.412023:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574689.412025:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574689.412026:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788574689.412029:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574689.412030:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574689.412031:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788574689.412036:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574689.412037:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.412040:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574689.412043:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516729216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.412049:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574689.412055:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574689.412057:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574689.412061:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574689.412062:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574689.412064:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574689.412066:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574689.412067:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788574689.412070:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574689.412071:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574689.412072:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788574689.412075:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574689.412076:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.412078:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574689.412082:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516729344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574714 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574689.412088:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574689.412090:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574689.412091:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574689.412094:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574689.412095:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574689.412097:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574689.412099:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574689.412100:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788574689.412103:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574689.412104:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574689.412105:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788574689.412107:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574689.412108:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.412110:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.412114:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574689.412115:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.412117:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574689.412119:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.920878:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574689.920885:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574689.920887:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574689.920894:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574689.920916:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574689.920920:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.920927:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574689.920929:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.920934:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574689.920979:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574689.920984:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574689.920989:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574689.921006:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574689.921026:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574689.921032:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.921035:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574689.921036:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574689.921038:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574690.625146:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574690.625154:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574690.625158:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574690.944967:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574690.944974:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574690.944981:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574690.944989:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574690.944998:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574690.945002:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574690.945045:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574690.945049:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574690.945054:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574691.968913:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574691.968921:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574691.968930:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574691.968941:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574691.968950:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574691.968956:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574691.968961:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574691.968966:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574691.968973:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574691.969020:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574691.969025:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574691.969028:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574691.969046:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574691.969054:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574691.969075:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574691.969079:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574691.969080:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574691.969083:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574692.672972:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574692.672976:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574692.672980:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574692.992893:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574692.992901:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574692.992905:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574692.992911:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574692.992919:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574692.992926:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.992934:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574692.992940:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574692.992946:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574693.248898:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310053102, at 4310053120 00000100:00000010:0.0:1788574693.248912:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574693.697038:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574693.697042:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574693.697045:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574694.016926:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574694.016934:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574694.016938:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574694.016944:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574694.016953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574694.016959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574694.016979:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574694.016983:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574694.016993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574694.528920:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574694.528931:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574694.528935:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574694.528939:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574694.528966:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574694.528971:0:265537:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15381 00000400:00000200:0.0:1788574694.528979:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574694.528985:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574694.528988:0:265537:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574694.528991:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.528994:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574694.528996:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.528998:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574694.529002:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574694.529004:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574694.529006:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574694.529009:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574694.529011:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574694.529013:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574694.529015:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529016:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574694.529018:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529019:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574694.529022:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574694.529024:0:265537:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529026:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574694.529028:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574694.529030:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574694.529032:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529034:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574694.529036:0:265537:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574694.529041:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574694.529043:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574694.529046:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574694.529047:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529053:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574694.529055:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574694.529059:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574694.529065:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574694.529071:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574694.529079:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516729472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574694.529087:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574694.529091:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574694.529094:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516729472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574694.529118:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574694.529120:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574694.529123:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574694.529124:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574694.529125:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574694.529127:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574694.529129:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15381 00000400:00000200:0.0:1788574694.529132:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574694.529134:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574694.529136:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574694.529138:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529139:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574694.529141:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529142:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574694.529144:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574694.529146:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574694.529147:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574694.529148:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574694.529150:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574694.529152:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574694.529159:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529163:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574694.529165:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529169:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574694.529172:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574694.529176:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529178:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574694.529182:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574694.529187:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574694.529190:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529193:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574694.529198:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574694.529209:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529213:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574694.529217:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574694.529222:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574694.529226:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574694.529229:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529231:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574694.529233:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574694.529236:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574694.529240:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574694.529246:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574694.529254:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516729600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574694.529264:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574694.529267:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574694.529272:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516729600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574694.529284:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574694.529288:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574694.529299:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574694.529303:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574694.529305:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574694.529309:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574694.529314:0:265537:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15381 00000400:00000200:0.0:1788574694.529322:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574694.529328:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574694.529331:0:265537:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574694.529335:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529339:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574694.529342:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529346:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574694.529351:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574694.529352:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574694.529354:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574694.529357:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574694.529360:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574694.529364:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574694.529369:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529373:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574694.529378:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529381:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574694.529386:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574694.529388:0:265537:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529390:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574694.529391:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574694.529393:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574694.529396:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529399:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574694.529402:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574694.529410:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529413:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574694.529417:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574694.529421:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574694.529425:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574694.529427:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529429:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574694.529433:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574694.529434:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574694.529441:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574694.529445:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574694.529453:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516729728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574694.529465:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574694.529468:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574694.529474:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516729728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574694.529488:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574694.529493:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574694.529502:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574694.529505:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574694.529510:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574694.529514:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574694.529519:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15381 00000400:00000200:0.0:1788574694.529529:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574694.529534:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574694.529538:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574694.529545:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529549:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574694.529552:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529561:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574694.529564:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574694.529567:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574694.529570:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574694.529573:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574694.529577:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574694.529579:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574694.529586:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529587:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574694.529589:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529594:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574694.529595:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574694.529598:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529603:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574694.529604:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574694.529607:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574694.529610:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529614:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574694.529619:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574694.529625:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529630:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574694.529636:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574694.529642:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574694.529648:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574694.529652:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529657:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574694.529661:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574694.529665:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574694.529670:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574694.529678:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574694.529688:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516729856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574694.529725:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574694.529729:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574694.529736:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516729856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574694.529751:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574694.529754:0:265537:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574694.529756:0:265537:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574694.529758:0:265537:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574694.529761:0:265537:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574694.529766:0:265537:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574694.529771:0:265537:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15381 00000400:00000200:0.0:1788574694.529783:0:265537:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574694.529792:0:265537:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574694.529796:0:265537:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574694.529803:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529842:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574694.529844:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529846:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574694.529848:0:265537:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574694.529849:0:265537:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574694.529851:0:265537:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574694.529852:0:265537:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574694.529854:0:265537:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574694.529856:0:265537:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574694.529862:0:265537:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529866:0:265537:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574694.529868:0:265537:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529871:0:265537:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574694.529881:0:265537:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574694.529885:0:265537:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574694.529891:0:265537:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574694.529894:0:265537:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574694.529899:0:265537:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574694.529905:0:265537:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529911:0:265537:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574694.529915:0:265537:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574694.529921:0:265537:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529923:0:265537:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574694.529928:0:265537:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574694.529933:0:265537:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574694.529938:0:265537:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574694.529942:0:265537:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574694.529948:0:265537:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574694.529952:0:265537:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574694.529956:0:265537:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574694.529964:0:265537:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574694.529974:0:265537:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574694.529986:0:265537:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516729984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574694.530005:0:265537:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574694.530010:0:265537:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574694.530017:0:265537:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516729984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574694.530033:0:265537:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574694.530040:0:265537:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15392) 00000100:00000001:0.0:1788574695.040902:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574695.040907:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574695.040911:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574695.040916:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574695.040922:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574695.040937:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574695.040945:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574695.040951:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574695.040958:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574695.744984:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574695.744987:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574695.744990:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574696.064906:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574696.064912:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574696.064917:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574696.064922:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574696.064927:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574696.064931:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574696.064944:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574696.064949:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574696.064955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574697.088948:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574697.088952:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574697.088954:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574697.088957:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574697.088961:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574697.088963:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574697.088966:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574697.088969:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574697.088972:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574697.792984:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574697.792987:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574697.792990:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574698.112906:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574698.112914:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574698.112918:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574698.112930:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574698.112938:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574698.112944:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574698.112964:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574698.112968:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574698.112973:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574698.368913:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310058221, at 4310058240 00000100:00000010:0.0:1788574698.368940:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab5a0. 00000020:00000001:0.0:1788574698.816996:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574698.816999:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574698.817002:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574699.136949:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574699.136955:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574699.136959:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574699.136965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574699.136972:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574699.136976:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574699.136981:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574699.136984:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574699.136986:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574699.648900:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574699.648911:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574699.648915:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574699.648919:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574699.648924:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574699.648931:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15387 00000400:00000200:0.0:1788574699.648939:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574699.648945:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574699.648949:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574699.648953:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.648957:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574699.648960:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.648963:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574699.648968:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574699.648970:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574699.648973:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574699.648976:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574699.648979:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574699.648982:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574699.648986:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649005:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574699.649008:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649010:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574699.649014:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574699.649018:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649021:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574699.649024:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574699.649027:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574699.649031:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.649034:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574699.649036:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574699.649043:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574699.649047:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574699.649051:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574699.649053:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.649056:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574699.649058:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574699.649062:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574699.649069:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574699.649074:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574699.649082:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516730112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574699.649091:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574699.649096:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574699.649100:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516730112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574699.649135:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574699.649142:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574699.649146:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574699.649147:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574699.649148:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574699.649150:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574699.649157:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15387 00000400:00000200:0.0:1788574699.649161:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574699.649164:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574699.649166:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574699.649169:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649170:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574699.649172:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649174:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574699.649176:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574699.649177:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574699.649179:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574699.649180:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574699.649181:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574699.649184:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574699.649189:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649192:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574699.649195:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649198:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574699.649202:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574699.649206:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649208:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574699.649210:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574699.649216:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574699.649220:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.649223:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574699.649228:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574699.649231:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.649236:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574699.649239:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574699.649244:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574699.649249:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574699.649256:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.649259:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574699.649261:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574699.649264:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574699.649270:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574699.649276:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574699.649284:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516730240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574699.649296:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574699.649301:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574699.649309:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516730240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574699.649327:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574699.649332:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574699.649337:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574699.649341:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574699.649346:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574699.649350:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574699.649354:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15387 00000400:00000200:0.0:1788574699.649379:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574699.649387:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574699.649392:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574699.649399:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649405:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574699.649411:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649416:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574699.649421:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574699.649423:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574699.649426:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574699.649427:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574699.649429:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574699.649441:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574699.649448:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649450:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574699.649453:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649459:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574699.649471:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574699.649482:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649486:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574699.649490:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574699.649493:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574699.649497:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.649500:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574699.649503:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574699.649508:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.649510:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574699.649514:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574699.649517:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574699.649521:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574699.649523:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.649526:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574699.649529:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574699.649532:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574699.649536:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574699.649540:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574699.649547:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516730368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574699.649556:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574699.649560:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574699.649567:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516730368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574699.649581:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574699.649591:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574699.649595:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574699.649597:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574699.649601:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574699.649608:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574699.649616:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15387 00000400:00000200:0.0:1788574699.649625:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574699.649632:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574699.649636:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574699.649641:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649646:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574699.649652:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649658:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574699.649664:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574699.649667:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574699.649672:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574699.649677:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574699.649683:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574699.649685:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574699.649690:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649694:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574699.649700:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649706:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574699.649709:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574699.649715:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649721:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574699.649726:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574699.649732:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574699.649737:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.649740:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574699.649751:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574699.649758:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.649763:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574699.649771:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574699.649775:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574699.649779:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574699.649781:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.649785:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574699.649786:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574699.649788:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574699.649796:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574699.649800:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574699.649827:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516730496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574699.649834:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574699.649835:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574699.649840:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516730496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574699.649854:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574699.649858:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574699.649867:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574699.649871:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574699.649874:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574699.649881:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574699.649887:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15387 00000400:00000200:0.0:1788574699.649898:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574699.649906:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574699.649910:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574699.649918:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649923:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574699.649927:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649940:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574699.649946:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574699.649951:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574699.649955:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574699.649961:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574699.649967:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574699.649972:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574699.649982:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649986:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574699.649992:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574699.649997:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574699.650001:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574699.650006:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574699.650011:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574699.650016:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574699.650021:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574699.650027:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.650032:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574699.650035:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574699.650042:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.650048:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574699.650057:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574699.650063:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574699.650068:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574699.650072:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574699.650076:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574699.650081:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574699.650086:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574699.650095:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574699.650102:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574699.650112:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516730624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574699.650150:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574699.650155:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574699.650165:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516730624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574699.650186:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574699.650196:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15397) 00000100:00000001:0.0:1788574700.160915:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574700.160928:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574700.160933:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574700.160940:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574700.160947:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574700.160952:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574700.160962:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574700.160968:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574700.160972:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574700.865075:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574700.865080:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574700.865084:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574701.184951:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574701.184966:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574701.184970:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574701.184975:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574701.184981:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574701.184986:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574701.184997:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574701.185001:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574701.185005:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.209003:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574702.209013:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574702.209018:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574702.209029:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574702.209034:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574702.209040:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.209062:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574702.209073:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.209078:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.721766:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574702.721772:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574702.721775:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574702.721784:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574699/real 1788574702] req@ffff8fc4b80c2a00 x1875440516730496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574702.721798:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574702.721800:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574702.721803:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a0790. 00000400:00000200:0.0:1788574702.721863:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed440 00000400:00000010:0.0:1788574702.721866:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed440. 00000100:00000001:0.0:1788574702.721869:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574702.721874:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c2a00 x1875440516730496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574702.721882:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c2a00 x1875440516730496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.721890:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574702.721892:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.721895:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574702.721897:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574702.721902:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c2a00 x1875440516730496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.721909:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574702.721915:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c2a00 x1875440516730496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.721924:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574702.721927:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.721929:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574702.721930:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.721933:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574702.721943:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574702.721947:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574702.721952:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574702.721955:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574702.721960:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c2a00 x1875440516730496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574702.721969:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516730496:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574702.721975:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574702.721979:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574699/real 1788574702] req@ffff8fc4b80c3800 x1875440516730624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574702.721988:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574702.721989:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574702.721991:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a07e8. 00000400:00000200:0.0:1788574702.721994:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed4c8 00000400:00000010:0.0:1788574702.721996:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed4c8. 00000100:00000001:0.0:1788574702.721999:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574702.722003:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c3800 x1875440516730624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574702.722011:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c3800 x1875440516730624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722018:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574702.722020:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722021:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574702.722023:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574702.722027:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c3800 x1875440516730624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722034:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574702.722038:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c3800 x1875440516730624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722056:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574702.722059:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722061:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574702.722063:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722065:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574702.722067:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574702.722070:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574702.722077:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574702.722081:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574702.722102:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c3800 x1875440516730624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574702.722113:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516730624:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574702.722119:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574702.722124:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574699/real 1788574702] req@ffff8fc4b80c3480 x1875440516730240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574702.722134:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574702.722136:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574702.722138:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a04d0. 00000400:00000200:0.0:1788574702.722147:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed908 00000400:00000010:0.0:1788574702.722150:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed908. 00000100:00000001:0.0:1788574702.722153:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574702.722158:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c3480 x1875440516730240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574702.722166:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c3480 x1875440516730240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722173:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574702.722181:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722184:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574702.722185:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574702.722190:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c3480 x1875440516730240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722198:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574702.722203:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c3480 x1875440516730240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722210:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574702.722212:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722214:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574702.722216:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722218:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574702.722221:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574702.722224:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574702.722228:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574702.722231:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574702.722235:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c3480 x1875440516730240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574702.722243:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516730240:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574702.722248:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574702.722252:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574699/real 1788574702] req@ffff8fc4b80c0000 x1875440516730368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574702.722260:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574702.722262:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574702.722264:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a06e0. 00000400:00000200:0.0:1788574702.722266:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76edb28 00000400:00000010:0.0:1788574702.722268:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76edb28. 00000100:00000001:0.0:1788574702.722276:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574702.722281:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c0000 x1875440516730368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574702.722289:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c0000 x1875440516730368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722296:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574702.722298:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722300:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574702.722301:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574702.722306:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c0000 x1875440516730368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722313:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574702.722317:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c0000 x1875440516730368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722324:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574702.722327:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722328:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574702.722329:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722332:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574702.722335:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574702.722337:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574702.722340:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574702.722343:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574702.722346:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c0000 x1875440516730368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574702.722354:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516730368:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574702.722359:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574702.722368:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574699/real 1788574702] req@ffff8fc4b80c1880 x1875440516730112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574702.722376:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574702.722377:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574702.722380:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4ac8a0210. 00000400:00000200:0.0:1788574702.722382:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a76ed5d8 00000400:00000010:0.0:1788574702.722385:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a76ed5d8. 00000100:00000001:0.0:1788574702.722387:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574702.722391:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c1880 x1875440516730112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574702.722399:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c1880 x1875440516730112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722406:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574702.722407:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722409:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574702.722411:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574702.722415:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c1880 x1875440516730112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722422:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574702.722426:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c1880 x1875440516730112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722433:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574702.722435:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722437:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574702.722438:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722440:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574702.722442:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574702.722444:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574702.722448:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574702.722450:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574702.722460:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c1880 x1875440516730112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574702.722468:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516730112:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574702.722472:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722475:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574702.722479:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c2a00 x1875440516730496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722486:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574702.722489:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574702.722494:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788574702.722498:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574702.722501:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574702.722503:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574702.722506:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574702.722509:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574702.722513:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574702.722515:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574702.722517:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c2a00. 00000100:00000001:0.0:1788574702.722520:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574702.722522:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722524:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574702.722527:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c3800 x1875440516730624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722534:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574702.722541:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574702.722543:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788574702.722545:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574702.722547:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574702.722548:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574702.722550:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574702.722552:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574702.722554:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574702.722560:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574702.722562:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c3800. 00000100:00000001:0.0:1788574702.722564:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574702.722566:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722568:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574702.722572:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c3480 x1875440516730240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722578:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574702.722580:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574702.722582:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b862ac00. 02000000:00000001:0.0:1788574702.722584:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574702.722586:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574702.722587:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574702.722589:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574702.722591:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574702.722593:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574702.722594:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574702.722595:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c3480. 00000100:00000001:0.0:1788574702.722598:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574702.722599:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722601:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574702.722604:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c0000 x1875440516730368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722610:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574702.722611:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574702.722613:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788574702.722615:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574702.722617:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574702.722618:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574702.722620:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574702.722621:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574702.722624:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574702.722625:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574702.722626:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c0000. 00000100:00000001:0.0:1788574702.722629:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574702.722630:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722637:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574702.722641:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c1880 x1875440516730112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574702.722648:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574702.722650:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574702.722652:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788574702.722655:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574702.722656:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574702.722657:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574702.722659:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574702.722660:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574702.722663:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574702.722664:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574702.722665:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c1880. 00000100:00000001:0.0:1788574702.722667:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574702.722669:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722671:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574702.722678:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574702.722681:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574702.722683:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574702.722685:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574702.913107:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574702.913115:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574702.913119:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574703.232916:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574703.232927:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574703.232933:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574703.232942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574703.232955:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574703.232960:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574703.232969:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574703.232975:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574703.232982:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574703.488937:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310063341, at 4310063360 00000100:00000010:0.0:1788574703.488950:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab360. 00000020:00000001:0.0:1788574703.937107:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574703.937131:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574703.937134:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574704.257020:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574704.257027:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574704.257031:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574704.257038:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574704.257053:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574704.257062:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574704.257100:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574704.257103:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574704.257107:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574704.768924:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574704.768934:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574704.768937:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574704.768940:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574704.768944:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574704.768949:0:241211:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15392 00000400:00000200:0.0:1788574704.768972:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574704.768978:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574704.768981:0:241211:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574704.768984:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.768986:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.768989:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.768991:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574704.768994:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574704.768996:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574704.768998:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574704.769001:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574704.769003:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574704.769004:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574704.769006:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769008:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.769021:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769022:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574704.769025:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574704.769028:0:241211:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769029:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574704.769031:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574704.769034:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574704.769036:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.769037:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574704.769039:0:241211:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574704.769043:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574704.769055:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574704.769059:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574704.769061:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.769064:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574704.769067:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574704.769072:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574704.769088:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574704.769096:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574704.769105:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516730752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574704.769139:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574704.769144:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574704.769150:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516730752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574704.769215:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574704.769219:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574704.769231:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574704.769232:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574704.769234:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574704.769237:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574704.769247:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15392 00000400:00000200:0.0:1788574704.769252:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574704.769257:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574704.769260:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574704.769272:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769275:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.769277:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769281:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574704.769284:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574704.769288:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574704.769291:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574704.769295:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574704.769306:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574704.769313:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574704.769322:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769327:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.769339:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769343:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574704.769345:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574704.769353:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769360:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574704.769366:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574704.769383:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574704.769390:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.769394:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574704.769402:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574704.769416:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.769419:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574704.769427:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574704.769431:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574704.769438:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574704.769459:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.769464:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574704.769468:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574704.769472:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574704.769487:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574704.769497:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574704.769509:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516730880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574704.769532:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574704.769537:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574704.769551:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516730880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574704.769574:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574704.769580:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574704.769597:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574704.769602:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574704.769604:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574704.769609:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574704.769613:0:241211:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15392 00000400:00000200:0.0:1788574704.769630:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574704.769638:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574704.769645:0:241211:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574704.769651:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769665:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.769674:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769680:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574704.769685:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574704.769689:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574704.769702:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574704.769707:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574704.769713:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574704.769730:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574704.769747:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769754:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.769761:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769768:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574704.769780:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574704.769785:0:241211:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574704.769790:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574704.769794:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574704.769800:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574704.769842:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.769845:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574704.769848:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574704.769852:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.769857:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574704.769862:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574704.769867:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574704.769878:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574704.769883:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.769888:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574704.769892:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574704.769895:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574704.769924:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574704.769931:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574704.769943:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516731008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574704.769977:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574704.769983:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574704.770001:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516731008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574704.770037:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574704.770062:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574704.770073:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574704.770084:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574704.770089:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574704.770097:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574704.770104:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15392 00000400:00000200:0.0:1788574704.770119:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574704.770127:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574704.770134:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574704.770144:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.770158:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.770164:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.770169:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574704.770175:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574704.770178:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574704.770182:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574704.770186:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574704.770198:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574704.770203:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574704.770212:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.770220:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.770233:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.770239:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574704.770245:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574704.770254:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574704.770267:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574704.770272:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574704.770279:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574704.770284:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.770288:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574704.770306:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574704.770312:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.770319:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574704.770327:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574704.770334:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574704.770350:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574704.770352:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.770354:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574704.770357:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574704.770361:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574704.770377:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574704.770385:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574704.770398:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516731136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574704.770422:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574704.770428:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574704.770436:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516731136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574704.770459:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574704.770463:0:241211:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574704.770468:0:241211:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574704.770475:0:241211:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574704.770479:0:241211:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574704.770488:0:241211:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574704.770495:0:241211:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15392 00000400:00000200:0.0:1788574704.770514:0:241211:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574704.770524:0:241211:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574704.770529:0:241211:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574704.770538:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.770550:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.770556:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.770566:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574704.770570:0:241211:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574704.770573:0:241211:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574704.770581:0:241211:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574704.770585:0:241211:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574704.770591:0:241211:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574704.770593:0:241211:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574704.770605:0:241211:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.770611:0:241211:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574704.770623:0:241211:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574704.770631:0:241211:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574704.770637:0:241211:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574704.770646:0:241211:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574704.770659:0:241211:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574704.770665:0:241211:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574704.770671:0:241211:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574704.770676:0:241211:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.770682:0:241211:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574704.770693:0:241211:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574704.770700:0:241211:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.770704:0:241211:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574704.770735:0:241211:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574704.770742:0:241211:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574704.770746:0:241211:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574704.770747:0:241211:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574704.770749:0:241211:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574704.770754:0:241211:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574704.770757:0:241211:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574704.770767:0:241211:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574704.770775:0:241211:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574704.770787:0:241211:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516731264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574704.770894:0:241211:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574704.770924:0:241211:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574704.770936:0:241211:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516731264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574704.770978:0:241211:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574704.770985:0:241211:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15402) 00000020:00000001:0.0:1788574705.985036:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574705.985040:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574705.985043:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574706.304920:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574706.304927:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574706.304933:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574706.304940:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574706.304948:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574706.304953:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574706.304966:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574706.304971:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574706.304974:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574707.328962:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574707.328969:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574707.328975:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574707.328988:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574707.328995:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574707.328999:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574707.329003:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574707.329006:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574707.329009:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574708.032992:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574708.032998:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574708.033003:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574708.352913:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574708.352918:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574708.352922:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574708.352928:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574708.352949:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574708.352953:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574708.352962:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574708.352965:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574708.352968:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574708.608898:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310068461, at 4310068480 00000100:00000010:0.0:1788574708.608914:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab8a0. 00000020:00000001:0.0:1788574709.056989:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574709.056992:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574709.056995:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574709.376936:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574709.376947:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574709.376961:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574709.376973:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574709.376985:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574709.376992:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.377020:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574709.377023:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.377026:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.889205:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574709.889211:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574709.889214:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574709.889223:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516731392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.889235:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574709.889238:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.889242:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574709.889245:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574709.889249:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574709.889256:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516731392:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574709.889314:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574709.889318:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574709.889321:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.889327:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574709.889333:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574709.889345:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574709.889353:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574709.889364:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574709.889372:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574709.889381:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516731392, portal 25 00000100:00000001:0.0:1788574709.889386:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574709.889388:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788574709.889396:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516731392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.889412:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574709.889417:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574709.889421:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574709.889426:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516731392, offset 0 00000400:00000200:0.0:1788574709.889433:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574709.889454:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574709.889461:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574709.889466:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574709.889469:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574709.889472:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574709.889475:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574709.889478:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547366:547366:256:4294967295] 192.168.202.128@tcp LPNI seq info [547289:547289:8:4294967295] 00000400:00000200:0.0:1788574709.889484:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574709.889504:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574709.889511:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574709.889518:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574709.889528:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574709.889534:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574709.889558:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574709.889561:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574709.889567:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574709.889575:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574709.889588:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574709.889592:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574709.889598:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574709.889605:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516731392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.889618:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574709.889624:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516731392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.889640:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.889644:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574709.889649:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574709.889655:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574709.889659:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310073581 00000100:00000010:0.0:1788574709.889667:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574709.889675:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574709.889681:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310073581 00000100:00000010:0.0:1788574709.889686:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50d80. 00000100:00000001:0.0:1788574709.889691:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574709.889695:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574709.889698:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.889701:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574709.889707:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574709/real 1788574709] req@ffff8fc4ac821880 x1875440516731392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574709.889721:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574709.889724:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574709.889727:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574709.889731:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574709.889734:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574709.889737:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574709.889741:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516731392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574709.889753:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516731392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.889766:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574709.889768:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.889771:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574709.889774:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574709.889779:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516731392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.889787:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574709.889794:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516731392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.889802:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574709.889829:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.889831:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574709.889833:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.889837:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574709.889840:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574709.889845:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574709.889852:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574709.889856:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574709.889864:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516731392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574709.889874:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516731392:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574709.889878:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.889881:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574709.889885:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516731392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.889894:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574709.889897:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574709.889905:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574709.889909:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574709.889912:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574709.889915:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574709.889919:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574709.889921:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788574709.889926:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574709.889929:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574709.889932:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788574709.889936:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574709.889939:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.889942:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.889948:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574709.889952:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.889955:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574709.889959:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574709.889962:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574709.889968:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516731520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.889980:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574709.889982:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.889986:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574709.889989:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574709.889991:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574709.889993:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516731520:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574709.889995:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574709.889999:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574709.890002:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.890005:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574709.890010:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574709.890014:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574709.890023:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574709.890030:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574709.890039:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574709.890046:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516731520, portal 10 00000100:00000001:0.0:1788574709.890061:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574709.890066:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788574709.890079:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516731520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890090:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574709.890094:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574709.890099:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574709.890103:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516731520, offset 0 00000400:00000200:0.0:1788574709.890109:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574709.890117:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574709.890123:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574709.890126:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574709.890128:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574709.890131:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574709.890135:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574709.890139:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547367:547367:256:4294967295] 192.168.202.128@tcp LPNI seq info [547290:547290:8:4294967295] 00000400:00000200:0.0:1788574709.890146:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574709.890154:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574709.890160:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574709.890165:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574709.890172:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574709.890178:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574709.890195:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574709.890199:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574709.890204:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574709.890211:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574709.890215:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574709.890217:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574709.890223:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574709.890227:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516731520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890244:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574709.890247:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516731520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890252:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.890254:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574709.890257:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574709.890260:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574709.890263:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574709.890264:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310069779 00000100:00000001:0.0:1788574709.890266:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310073581 00000100:00000001:0.0:1788574709.890268:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310069779 00000100:00000001:0.0:1788574709.890270:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310073581 00000100:00000001:0.0:1788574709.890271:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574709.890272:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.890274:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574709.890276:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574709/real 1788574709] req@ffff8fc4ac820e00 x1875440516731520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574709.890282:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574709.890282:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574709.890284:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574709.890286:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574709.890287:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574709.890289:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574709.890293:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516731520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574709.890302:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516731520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890309:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574709.890316:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890318:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574709.890319:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574709.890323:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516731520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890330:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574709.890335:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516731520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890341:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574709.890344:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890345:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574709.890347:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890349:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574709.890350:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574709.890353:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574709.890356:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574709.890359:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574709.890363:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516731520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574709.890371:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516731520:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574709.890375:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574709.890378:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516731648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890382:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574709.890383:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.890385:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574709.890386:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574709.890387:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574709.890394:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516731648:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574709.890397:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574709.890398:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574709.890399:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.890401:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574709.890403:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574709.890404:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574709.890407:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574709.890409:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574709.890411:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574709.890414:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516731648, portal 10 00000100:00000001:0.0:1788574709.890415:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574709.890416:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788574709.890421:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516731648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890425:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574709.890426:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574709.890428:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574709.890429:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516731648, offset 0 00000400:00000200:0.0:1788574709.890432:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574709.890434:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574709.890450:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574709.890453:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574709.890455:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574709.890457:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574709.890459:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574709.890461:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547368:547368:256:4294967295] 192.168.202.128@tcp LPNI seq info [547291:547291:8:4294967295] 00000400:00000200:0.0:1788574709.890464:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574709.890469:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574709.890471:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574709.890474:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574709.890480:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574709.890482:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574709.890489:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574709.890490:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574709.890493:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574709.890495:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574709.890498:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574709.890499:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574709.890501:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574709.890504:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516731648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890509:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574709.890511:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516731648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890516:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.890517:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574709.890519:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574709.890521:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574709.890524:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574709.890525:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310069786 00000100:00000001:0.0:1788574709.890527:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310073581 00000100:00000001:0.0:1788574709.890528:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310069786 00000100:00000001:0.0:1788574709.890530:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310073581 00000100:00000001:0.0:1788574709.890531:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574709.890533:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.890534:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574709.890536:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574709/real 1788574709] req@ffff8fc4ac823800 x1875440516731648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574709.890541:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574709.890545:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574709.890546:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574709.890548:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574709.890549:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574709.890551:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574709.890553:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516731648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574709.890559:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516731648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890562:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574709.890563:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890565:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574709.890565:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574709.890568:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516731648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890572:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574709.890574:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516731648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890579:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574709.890580:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890582:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574709.890582:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890584:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574709.890585:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574709.890587:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574709.890589:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574709.890590:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574709.890592:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516731648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574709.890604:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516731648:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574709.890607:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574709.890610:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516731776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890614:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574709.890614:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.890616:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574709.890617:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574709.890618:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574709.890620:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516731776:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574709.890622:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574709.890623:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574709.890625:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.890626:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574709.890628:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574709.890629:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574709.890631:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574709.890633:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574709.890636:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574709.890638:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516731776, portal 4 00000100:00000001:0.0:1788574709.890639:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574709.890640:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788574709.890644:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516731776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890648:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574709.890650:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574709.890651:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574709.890653:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516731776, offset 0 00000400:00000200:0.0:1788574709.890655:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574709.890662:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574709.890664:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574709.890666:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574709.890668:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574709.890670:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574709.890671:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574709.890674:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547369:547369:256:4294967295] 192.168.202.128@tcp LPNI seq info [547292:547292:8:4294967295] 00000400:00000200:0.0:1788574709.890677:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574709.890680:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574709.890683:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574709.890685:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574709.890687:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574709.890689:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574709.890696:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574709.890698:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574709.890700:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574709.890702:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574709.890704:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574709.890705:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574709.890707:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574709.890709:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516731776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890714:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574709.890716:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516731776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890720:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.890721:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574709.890723:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574709.890725:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574709.890727:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574709.890732:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310069926 00000100:00000001:0.0:1788574709.890734:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310073581 00000100:00000001:0.0:1788574709.890735:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310069926 00000100:00000001:0.0:1788574709.890737:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310073581 00000100:00000001:0.0:1788574709.890738:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574709.890739:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.890740:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574709.890743:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574709/real 1788574709] req@ffff8fc4ac820a80 x1875440516731776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574709.890748:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574709.890749:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574709.890750:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574709.890752:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574709.890753:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574709.890754:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574709.890756:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516731776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574709.890761:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516731776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890765:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574709.890766:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890767:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574709.890768:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574709.890770:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516731776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890774:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574709.890777:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516731776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890781:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574709.890785:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890786:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574709.890787:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890788:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574709.890790:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574709.890791:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574709.890793:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574709.890795:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574709.890797:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516731776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574709.890802:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516731776:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574709.890804:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890806:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574709.890825:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516731520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890830:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574709.890832:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574709.890834:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574709.890836:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574709.890838:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574709.890839:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574709.890841:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574709.890843:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788574709.890845:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574709.890846:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574709.890847:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788574709.890849:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574709.890850:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890851:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574709.890854:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516731648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890866:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574709.890867:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574709.890868:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574709.890869:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574709.890870:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574709.890871:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574709.890872:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574709.890873:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788574709.890874:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574709.890875:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574709.890875:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788574709.890877:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574709.890878:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890879:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574709.890881:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516731776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890885:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574709.890885:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574709.890886:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574709.890887:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574709.890888:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574709.890889:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574709.890890:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574709.890890:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788574709.890892:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574709.890893:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574709.890894:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788574709.890895:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574709.890896:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890897:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.890900:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574709.890901:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.890902:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574709.890903:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574709.890904:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574709.890909:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516731904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890913:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574709.890914:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.890916:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574709.890917:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574709.890918:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574709.890920:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516731904:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574709.890922:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574709.890924:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574709.890925:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574709.890926:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574709.890928:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574709.890929:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574709.890932:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574709.890933:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574709.890936:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574709.890938:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516731904, portal 4 00000100:00000001:0.0:1788574709.890939:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574709.890940:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788574709.890944:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516731904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.890948:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574709.890950:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574709.890951:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574709.890953:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516731904, offset 0 00000400:00000200:0.0:1788574709.890955:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574709.890958:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574709.890960:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574709.890962:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574709.890963:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574709.890965:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574709.890967:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574709.890972:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547370:547370:256:4294967295] 192.168.202.128@tcp LPNI seq info [547293:547293:8:4294967295] 00000400:00000200:0.0:1788574709.890975:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574709.890979:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574709.890982:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574709.890984:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574709.890987:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574709.890989:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788574709.890999:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574709.891000:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574709.891010:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574709.891013:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574709.891020:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574709.891022:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574709.891025:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574709.891029:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516731904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.891037:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574709.891040:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516731904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574734 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.891047:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.891049:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574709.891053:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574709.891056:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574709.891059:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574709.891062:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310069929 00000100:00000001:0.0:1788574709.891065:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310073581 00000100:00000001:0.0:1788574709.891067:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310069929 00000100:00000001:0.0:1788574709.891069:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310073581 00000100:00000001:0.0:1788574709.891071:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574709.891079:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.891081:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574709.891084:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574709/real 1788574709] req@ffff8fc4ac821c00 x1875440516731904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574709.891091:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574709.891091:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574709.891093:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574709.891095:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574709.891096:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574709.891098:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574709.891100:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516731904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574709.891105:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516731904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.891109:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574709.891110:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.891111:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574709.891112:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574709.891114:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516731904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.891119:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574709.891121:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516731904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.891126:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574709.891127:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.891128:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574709.891129:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.891130:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574709.891131:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574709.891133:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574709.891138:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574709.891139:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574709.891142:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516731904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574709.891147:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516731904:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574709.891150:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.891151:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574709.891153:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516731904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574709.891157:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574709.891158:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574709.891159:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574709.891160:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574709.891161:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574709.891162:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574709.891163:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574709.891164:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574709.891166:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574709.891166:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574709.891167:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788574709.891169:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574709.891170:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.891171:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574709.891174:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574709.891175:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574709.891176:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574709.891177:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574710.400945:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574710.400952:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574710.400955:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574710.400961:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574710.400965:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574710.400971:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574710.401003:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574710.401006:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574710.401016:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574711.105046:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574711.105050:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574711.105053:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574711.424900:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574711.424909:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574711.424915:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574711.424924:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574711.424929:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574711.424933:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574711.424936:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574711.424939:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574711.424943:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574711.424993:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574711.424998:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574711.425007:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574711.425021:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574711.425031:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574711.425036:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574711.425040:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574711.425045:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574711.425049:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574712.448895:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574712.448902:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574712.448907:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574712.448916:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574712.448924:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574712.448929:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574712.448932:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574712.448935:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574712.448938:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574713.152996:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574713.153000:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574713.153004:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574713.472946:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574713.472953:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574713.472972:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574713.472977:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574713.472982:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574713.472986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574713.472997:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574713.473000:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574713.473004:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574713.728887:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310073581, at 4310073600 00000100:00000010:0.0:1788574713.728906:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574714.176984:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574714.176987:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574714.176990:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574714.496876:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574714.496883:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574714.496887:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574714.496897:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574714.496905:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574714.496911:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574714.496922:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574714.496924:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574714.496927:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574714.496961:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574714.496965:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574714.496967:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574714.496969:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574714.496971:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574714.496973:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574714.496974:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574714.496976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574714.496977:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.009222:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574715.009228:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574715.009232:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574715.009242:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516732032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.009260:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574715.009271:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574715.009277:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574715.009279:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574715.009284:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574715.009291:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516732032:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574715.009297:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574715.009303:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574715.009307:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574715.009311:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574715.009316:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574715.009321:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574715.009329:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574715.009335:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574715.009343:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574715.009351:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516732032, portal 25 00000100:00000001:0.0:1788574715.009357:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574715.009359:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788574715.009372:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516732032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.009398:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574715.009407:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574715.009414:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574715.009421:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516732032, offset 0 00000400:00000200:0.0:1788574715.009435:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574715.009446:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574715.009454:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574715.009470:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574715.009476:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574715.009482:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574715.009488:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574715.009496:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547371:547371:256:4294967295] 192.168.202.128@tcp LPNI seq info [547294:547294:8:4294967295] 00000400:00000200:0.0:1788574715.009505:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574715.009519:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574715.009536:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574715.009544:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574715.009557:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574715.009564:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574715.009607:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574715.009613:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574715.009624:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574715.009637:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574715.009647:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574715.009652:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574715.009658:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574715.009665:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516732032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.009683:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574715.009689:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516732032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.009701:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.009704:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574715.009711:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574715.009720:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574715.009725:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310078701 00000100:00000010:0.0:1788574715.009731:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574715.009739:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574715.009747:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310078701 00000100:00000010:0.0:1788574715.009752:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b508a0. 00000100:00000001:0.0:1788574715.009757:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574715.009762:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574715.009767:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.009770:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574715.009778:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574715/real 1788574715] req@ffff8fc4901a5880 x1875440516732032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574715.009830:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574715.009832:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574715.009834:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574715.009839:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574715.009843:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574715.009849:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574715.009854:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516732032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574715.009876:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516732032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.009889:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574715.009893:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.009905:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574715.009911:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574715.009919:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516732032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.009930:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574715.009938:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516732032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.009946:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574715.009952:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.009956:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574715.009959:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.009963:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574715.009966:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574715.009970:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574715.009988:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574715.009993:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574715.010000:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516732032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574715.010015:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516732032:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574715.010020:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010021:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574715.010023:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516732032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010028:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574715.010030:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574715.010032:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574715.010034:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574715.010035:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574715.010036:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574715.010038:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574715.010040:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788574715.010043:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574715.010045:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574715.010046:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788574715.010050:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574715.010051:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010053:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010058:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574715.010059:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.010061:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574715.010062:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574715.010064:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574715.010066:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516732160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010071:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574715.010072:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574715.010073:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574715.010075:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574715.010076:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574715.010078:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516732160:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574715.010083:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574715.010085:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574715.010086:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574715.010088:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574715.010089:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574715.010091:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574715.010094:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574715.010096:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574715.010099:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574715.010101:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516732160, portal 10 00000100:00000001:0.0:1788574715.010102:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574715.010103:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788574715.010108:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516732160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010113:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574715.010114:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574715.010116:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574715.010117:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516732160, offset 0 00000400:00000200:0.0:1788574715.010120:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574715.010123:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574715.010125:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574715.010128:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574715.010130:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574715.010132:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574715.010134:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574715.010136:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547372:547372:256:4294967295] 192.168.202.128@tcp LPNI seq info [547295:547295:8:4294967295] 00000400:00000200:0.0:1788574715.010139:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574715.010144:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574715.010147:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574715.010149:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574715.010152:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574715.010169:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574715.010181:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574715.010183:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574715.010186:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574715.010190:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574715.010194:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574715.010196:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574715.010199:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574715.010203:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516732160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010213:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574715.010216:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516732160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010222:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.010223:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574715.010226:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574715.010229:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574715.010232:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574715.010233:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310074899 00000100:00000001:0.0:1788574715.010235:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310078701 00000100:00000001:0.0:1788574715.010237:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310074899 00000100:00000001:0.0:1788574715.010239:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310078701 00000100:00000001:0.0:1788574715.010240:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574715.010242:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.010243:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574715.010246:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574715/real 1788574715] req@ffff8fc4901a4700 x1875440516732160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574715.010251:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574715.010252:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574715.010258:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574715.010259:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574715.010260:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574715.010263:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574715.010265:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516732160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574715.010271:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516732160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010275:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574715.010276:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010277:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574715.010278:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574715.010280:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516732160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010284:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574715.010287:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516732160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010291:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574715.010292:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010293:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574715.010294:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010295:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574715.010296:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574715.010298:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574715.010301:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574715.010302:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574715.010304:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516732160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574715.010309:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516732160:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574715.010315:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574715.010318:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516732288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010322:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574715.010323:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574715.010324:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574715.010326:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574715.010327:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574715.010329:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516732288:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574715.010331:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574715.010332:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574715.010334:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574715.010335:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574715.010337:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574715.010338:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574715.010341:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574715.010342:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574715.010344:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574715.010346:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516732288, portal 10 00000100:00000001:0.0:1788574715.010348:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574715.010349:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788574715.010353:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516732288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010357:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574715.010359:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574715.010360:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574715.010362:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516732288, offset 0 00000400:00000200:0.0:1788574715.010364:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574715.010367:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574715.010369:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574715.010371:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574715.010377:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574715.010380:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574715.010383:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574715.010386:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547373:547373:256:4294967295] 192.168.202.128@tcp LPNI seq info [547296:547296:8:4294967295] 00000400:00000200:0.0:1788574715.010390:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574715.010396:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574715.010401:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574715.010405:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574715.010409:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574715.010412:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574715.010424:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574715.010426:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574715.010429:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574715.010431:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574715.010434:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574715.010435:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574715.010437:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574715.010440:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516732288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010445:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574715.010447:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516732288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010452:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.010453:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574715.010455:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574715.010457:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574715.010459:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574715.010461:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310074906 00000100:00000001:0.0:1788574715.010463:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310078701 00000100:00000001:0.0:1788574715.010464:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310074906 00000100:00000001:0.0:1788574715.010471:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310078701 00000100:00000001:0.0:1788574715.010473:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574715.010474:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.010475:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574715.010478:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574715/real 1788574715] req@ffff8fc4901a7480 x1875440516732288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574715.010484:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574715.010485:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574715.010486:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574715.010487:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574715.010489:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574715.010490:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574715.010492:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516732288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574715.010497:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516732288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010501:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574715.010502:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010503:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574715.010504:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574715.010506:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516732288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010510:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574715.010513:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516732288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010518:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574715.010519:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010520:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574715.010521:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010525:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574715.010527:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574715.010529:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574715.010531:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574715.010533:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574715.010535:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516732288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574715.010540:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516732288:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574715.010543:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574715.010545:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516732416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010549:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574715.010551:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574715.010553:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574715.010555:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574715.010557:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574715.010559:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516732416:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574715.010562:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574715.010564:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574715.010566:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574715.010567:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574715.010571:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574715.010573:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574715.010577:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574715.010580:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574715.010584:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574715.010587:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516732416, portal 4 00000100:00000001:0.0:1788574715.010590:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574715.010591:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788574715.010598:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516732416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010610:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574715.010612:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574715.010614:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574715.010616:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516732416, offset 0 00000400:00000200:0.0:1788574715.010619:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574715.010622:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574715.010625:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574715.010627:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574715.010629:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574715.010630:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574715.010632:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574715.010634:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547374:547374:256:4294967295] 192.168.202.128@tcp LPNI seq info [547297:547297:8:4294967295] 00000400:00000200:0.0:1788574715.010637:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574715.010641:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574715.010644:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574715.010646:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574715.010649:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574715.010651:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574715.010658:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574715.010659:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574715.010662:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574715.010664:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574715.010666:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574715.010667:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574715.010669:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574715.010671:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516732416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010677:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574715.010679:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516732416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010691:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.010692:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574715.010695:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574715.010697:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574715.010699:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574715.010701:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310075046 00000100:00000001:0.0:1788574715.010702:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310078701 00000100:00000001:0.0:1788574715.010704:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310075046 00000100:00000001:0.0:1788574715.010705:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310078701 00000100:00000001:0.0:1788574715.010706:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574715.010708:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.010709:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574715.010711:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574715/real 1788574715] req@ffff8fc4901a6300 x1875440516732416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574715.010716:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574715.010717:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574715.010719:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574715.010720:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574715.010721:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574715.010723:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574715.010725:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516732416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574715.010730:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516732416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010734:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574715.010735:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010736:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574715.010737:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574715.010739:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516732416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010746:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574715.010749:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516732416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010753:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574715.010755:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010756:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574715.010757:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010758:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574715.010759:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574715.010761:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574715.010763:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574715.010764:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574715.010766:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516732416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574715.010771:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516732416:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574715.010774:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574715.010776:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516732544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010780:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574715.010780:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574715.010782:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574715.010783:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574715.010784:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574715.010786:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516732544:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574715.010788:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574715.010789:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574715.010792:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574715.010794:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574715.010796:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574715.010797:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574715.010799:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574715.010801:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574715.010803:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574715.010805:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516732544, portal 4 00000100:00000001:0.0:1788574715.010823:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574715.010824:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788574715.010829:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516732544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010833:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574715.010834:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574715.010836:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574715.010838:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516732544, offset 0 00000400:00000200:0.0:1788574715.010841:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574715.010843:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574715.010846:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574715.010848:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574715.010850:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574715.010851:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574715.010853:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574715.010856:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547375:547375:256:4294967295] 192.168.202.128@tcp LPNI seq info [547298:547298:8:4294967295] 00000400:00000200:0.0:1788574715.010858:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574715.010862:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574715.010865:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574715.010867:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574715.010870:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574715.010872:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788574715.010879:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574715.010884:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574715.010886:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574715.010889:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574715.010891:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574715.010892:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574715.010894:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574715.010897:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516732544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010902:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574715.010904:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516732544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010908:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.010909:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574715.010911:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574715.010914:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574715.010916:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574715.010917:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310075049 00000100:00000001:0.0:1788574715.010919:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310078701 00000100:00000001:0.0:1788574715.010920:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310075049 00000100:00000001:0.0:1788574715.010922:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310078701 00000100:00000001:0.0:1788574715.010923:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574715.010924:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.010925:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574715.010927:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574715/real 1788574715] req@ffff8fc4901a4e00 x1875440516732544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574715.010932:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574715.010933:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574715.010934:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574715.010936:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574715.010937:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574715.010942:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574715.010945:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516732544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574715.010950:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516732544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010954:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574715.010955:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010956:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574715.010957:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574715.010959:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516732544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010964:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574715.010966:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516732544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.010970:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574715.010972:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010985:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574715.010986:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.010988:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574715.010989:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574715.010991:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574715.010993:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574715.010994:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574715.010997:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516732544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574715.011002:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516732544:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574715.011005:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.011006:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574715.011012:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516732160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.011016:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574715.011018:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574715.011019:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574715.011021:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574715.011022:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574715.011023:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574715.011024:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574715.011026:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788574715.011027:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574715.011028:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574715.011029:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788574715.011032:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574715.011033:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.011034:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574715.011036:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516732288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.011041:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574715.011042:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574715.011043:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574715.011044:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574715.011045:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574715.011046:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574715.011047:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574715.011048:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788574715.011049:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574715.011049:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574715.011050:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788574715.011052:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574715.011053:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.011054:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574715.011056:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516732416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.011060:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574715.011063:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574715.011064:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574715.011066:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574715.011067:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574715.011068:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574715.011069:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574715.011070:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788574715.011072:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574715.011072:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574715.011073:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788574715.011076:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574715.011077:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.011078:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574715.011080:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516732544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574715.011084:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574715.011084:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574715.011085:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574715.011086:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574715.011087:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574715.011088:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574715.011090:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574715.011091:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788574715.011092:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574715.011093:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574715.011094:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788574715.011095:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574715.011096:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.011097:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.011100:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574715.011101:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.011102:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574715.011103:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574715.520901:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574715.520911:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574715.520914:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574715.520921:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574715.520952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574715.520957:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.520987:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574715.520991:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574715.520994:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574716.225028:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574716.225032:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574716.225034:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574716.544916:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574716.544925:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574716.544930:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574716.544938:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574716.544945:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574716.544950:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574716.544972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574716.544974:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574716.544977:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574718.273054:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574718.273058:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574718.273060:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574718.592906:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574718.592912:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574718.592915:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574718.592920:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574718.592924:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574718.592927:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574718.592929:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574718.592931:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574718.592933:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574718.592959:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574718.592963:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574718.592965:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574718.592969:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574718.592988:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574718.592998:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574718.593002:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574718.593003:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574718.593006:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574718.848883:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310078701, at 4310078720 00000100:00000010:0.0:1788574718.848894:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574719.296954:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574719.296957:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574719.296959:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574719.616915:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574719.616924:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574719.616929:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574719.616937:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574719.616945:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574719.616951:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574719.616972:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574719.616976:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574719.616980:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.129168:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574720.129173:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574720.129178:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574720.129189:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5180 x1875440516732672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.129211:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574720.129214:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574720.129217:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574720.129221:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574720.129226:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574720.129232:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516732672:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574720.129242:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574720.129249:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574720.129255:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574720.129262:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574720.129271:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574720.129275:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574720.129286:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574720.129292:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574720.129301:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574720.129308:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516732672, portal 25 00000100:00000001:0.0:1788574720.129320:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574720.129323:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605325184 : -123400104226432 : ffff8fc4ac9f5180) 00000100:00000040:0.0:1788574720.129329:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5180 x1875440516732672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.129337:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574720.129344:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574720.129349:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574720.129352:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516732672, offset 0 00000400:00000200:0.0:1788574720.129359:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574720.129370:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574720.129379:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574720.129384:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574720.129387:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574720.129390:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574720.129392:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574720.129395:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547376:547376:256:4294967295] 192.168.202.128@tcp LPNI seq info [547299:547299:8:4294967295] 00000400:00000200:0.0:1788574720.129398:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574720.129405:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574720.129413:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574720.129419:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574720.129429:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574720.129436:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574720.129460:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574720.129464:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574720.129471:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574720.129483:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574720.129488:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574720.129489:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574720.129494:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574720.129501:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5180 x1875440516732672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.129550:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574720.129557:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5180 x1875440516732672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.129572:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.129577:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574720.129584:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574720.129594:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574720.129597:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310083821 00000100:00000010:0.0:1788574720.129603:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574720.129613:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574720.129621:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310083821 00000100:00000010:0.0:1788574720.129626:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50120. 00000100:00000001:0.0:1788574720.129632:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574720.129636:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574720.129641:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.129645:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574720.129655:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574720/real 1788574720] req@ffff8fc4ac9f5180 x1875440516732672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574720.129682:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574720.129687:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574720.129693:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574720.129700:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574720.129704:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574720.129710:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574720.129718:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516732672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574720.129737:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516732672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.129753:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574720.129763:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.129768:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574720.129771:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574720.129778:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516732672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.129790:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574720.129798:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516732672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.129832:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574720.129835:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.129838:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574720.129839:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.129843:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574720.129848:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574720.129854:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574720.129859:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574720.129863:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574720.129871:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516732672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574720.129893:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516732672:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574720.129900:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.129904:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574720.129910:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516732672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.129931:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574720.129937:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574720.129944:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574720.129952:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574720.129957:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574720.129966:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574720.129972:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574720.129976:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35400. 02000000:00000001:0.0:1788574720.129980:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574720.129983:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574720.129986:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788574720.129991:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574720.129993:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.129997:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130003:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574720.130006:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.130009:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574720.130013:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574720.130015:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574720.130019:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7800 x1875440516732800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130028:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574720.130031:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574720.130036:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574720.130039:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574720.130044:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574720.130049:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516732800:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574720.130053:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574720.130055:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574720.130058:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574720.130061:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574720.130066:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574720.130070:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574720.130080:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574720.130085:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574720.130089:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574720.130097:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516732800, portal 10 00000100:00000001:0.0:1788574720.130103:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574720.130108:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335040 : -123400104216576 : ffff8fc4ac9f7800) 00000100:00000040:0.0:1788574720.130125:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7800 x1875440516732800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130148:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574720.130155:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574720.130161:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574720.130165:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516732800, offset 0 00000400:00000200:0.0:1788574720.130174:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574720.130180:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574720.130187:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574720.130194:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574720.130200:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574720.130203:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574720.130208:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574720.130211:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547377:547377:256:4294967295] 192.168.202.128@tcp LPNI seq info [547300:547300:8:4294967295] 00000400:00000200:0.0:1788574720.130217:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574720.130223:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574720.130227:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574720.130231:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574720.130235:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574720.130238:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574720.130249:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574720.130251:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574720.130254:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574720.130258:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574720.130262:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574720.130264:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574720.130267:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574720.130271:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7800 x1875440516732800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130280:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574720.130284:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7800 x1875440516732800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130299:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.130300:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574720.130303:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574720.130306:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574720.130308:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574720.130310:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310080019 00000100:00000001:0.0:1788574720.130312:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310083821 00000100:00000001:0.0:1788574720.130314:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310080019 00000100:00000001:0.0:1788574720.130315:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310083821 00000100:00000001:0.0:1788574720.130316:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574720.130318:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.130319:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574720.130321:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574720/real 1788574720] req@ffff8fc4ac9f7800 x1875440516732800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574720.130327:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574720.130328:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574720.130329:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574720.130331:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574720.130332:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574720.130334:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574720.130336:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7800 x1875440516732800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574720.130341:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7800 x1875440516732800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130346:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574720.130347:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130348:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574720.130349:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574720.130351:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7800 x1875440516732800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130358:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574720.130361:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7800 x1875440516732800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130366:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574720.130367:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130368:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574720.130369:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130370:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574720.130372:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574720.130373:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574720.130375:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574720.130376:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574720.130379:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7800 x1875440516732800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574720.130384:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516732800:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574720.130387:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574720.130389:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4700 x1875440516732928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130393:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574720.130394:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574720.130395:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574720.130396:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574720.130398:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574720.130399:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516732928:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574720.130401:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574720.130402:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574720.130406:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574720.130407:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574720.130410:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574720.130411:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574720.130413:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574720.130415:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574720.130427:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574720.130429:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516732928, portal 10 00000100:00000001:0.0:1788574720.130434:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574720.130435:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605322496 : -123400104229120 : ffff8fc4ac9f4700) 00000100:00000040:0.0:1788574720.130439:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4700 x1875440516732928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130444:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574720.130445:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574720.130447:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574720.130449:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516732928, offset 0 00000400:00000200:0.0:1788574720.130451:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574720.130454:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574720.130456:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574720.130458:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574720.130460:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574720.130462:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574720.130464:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574720.130466:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547378:547378:256:4294967295] 192.168.202.128@tcp LPNI seq info [547301:547301:8:4294967295] 00000400:00000200:0.0:1788574720.130468:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574720.130473:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574720.130476:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574720.130478:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574720.130481:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574720.130483:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574720.130490:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574720.130495:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574720.130497:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574720.130500:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574720.130502:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574720.130503:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574720.130505:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574720.130507:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4700 x1875440516732928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130525:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574720.130528:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4700 x1875440516732928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130532:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.130533:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574720.130536:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574720.130538:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574720.130541:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574720.130542:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310080026 00000100:00000001:0.0:1788574720.130544:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310083821 00000100:00000001:0.0:1788574720.130545:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310080026 00000100:00000001:0.0:1788574720.130547:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310083821 00000100:00000001:0.0:1788574720.130548:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574720.130549:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.130551:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574720.130553:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574720/real 1788574720] req@ffff8fc4ac9f4700 x1875440516732928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574720.130558:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574720.130559:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574720.130560:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574720.130562:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574720.130563:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574720.130569:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574720.130571:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4700 x1875440516732928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574720.130577:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4700 x1875440516732928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130581:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574720.130582:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130583:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574720.130584:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574720.130586:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4700 x1875440516732928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130590:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574720.130593:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4700 x1875440516732928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130597:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574720.130598:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130599:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574720.130600:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130601:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574720.130603:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574720.130605:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574720.130607:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574720.130608:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574720.130611:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4700 x1875440516732928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574720.130616:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516732928:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574720.130618:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574720.130623:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5f80 x1875440516733056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130627:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574720.130628:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574720.130630:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574720.130631:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574720.130633:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574720.130634:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516733056:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574720.130636:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574720.130638:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574720.130639:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574720.130640:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574720.130642:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574720.130644:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574720.130647:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574720.130650:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574720.130653:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574720.130657:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516733056, portal 4 00000100:00000001:0.0:1788574720.130659:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574720.130660:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605328768 : -123400104222848 : ffff8fc4ac9f5f80) 00000100:00000040:0.0:1788574720.130665:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5f80 x1875440516733056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130672:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574720.130675:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574720.130677:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574720.130680:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516733056, offset 0 00000400:00000200:0.0:1788574720.130684:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574720.130688:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574720.130692:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574720.130695:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574720.130698:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574720.130700:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574720.130702:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574720.130709:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547379:547379:256:4294967295] 192.168.202.128@tcp LPNI seq info [547302:547302:8:4294967295] 00000400:00000200:0.0:1788574720.130712:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574720.130717:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574720.130720:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574720.130723:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574720.130725:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574720.130727:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574720.130735:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574720.130736:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574720.130738:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574720.130741:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574720.130743:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574720.130744:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574720.130746:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574720.130748:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5f80 x1875440516733056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130754:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574720.130756:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5f80 x1875440516733056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130760:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.130761:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574720.130763:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574720.130765:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574720.130767:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574720.130768:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310080166 00000100:00000001:0.0:1788574720.130770:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310083821 00000100:00000001:0.0:1788574720.130772:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310080166 00000100:00000001:0.0:1788574720.130773:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310083821 00000100:00000001:0.0:1788574720.130775:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574720.130780:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.130781:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574720.130784:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574720/real 1788574720] req@ffff8fc4ac9f5f80 x1875440516733056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574720.130790:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574720.130791:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574720.130792:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574720.130794:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574720.130795:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574720.130796:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574720.130798:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5f80 x1875440516733056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574720.130803:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5f80 x1875440516733056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130824:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574720.130826:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130827:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574720.130828:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574720.130831:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5f80 x1875440516733056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130835:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574720.130838:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5f80 x1875440516733056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130842:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574720.130844:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130845:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574720.130846:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130847:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574720.130848:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574720.130850:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574720.130856:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574720.130857:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574720.130860:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5f80 x1875440516733056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574720.130865:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516733056:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574720.130868:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130869:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574720.130871:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7800 x1875440516732800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130875:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574720.130876:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574720.130878:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574720.130880:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574720.130881:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574720.130882:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574720.130884:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574720.130885:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788574720.130887:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574720.130887:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574720.130888:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7800. 00000100:00000001:0.0:1788574720.130890:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574720.130891:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130892:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574720.130894:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4700 x1875440516732928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130898:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574720.130899:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574720.130900:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574720.130901:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574720.130902:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574720.130902:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574720.130903:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574720.130907:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788574720.130909:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574720.130910:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574720.130911:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4700. 00000100:00000001:0.0:1788574720.130914:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574720.130915:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130917:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574720.130921:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5f80 x1875440516733056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130927:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574720.130928:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574720.130930:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574720.130932:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574720.130934:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574720.130935:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574720.130937:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574720.130939:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788574720.130941:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574720.130943:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574720.130944:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5f80. 00000100:00000001:0.0:1788574720.130947:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574720.130948:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130950:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.130954:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574720.130955:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.130957:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574720.130959:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574720.130960:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574720.130965:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516733184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.130970:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574720.130971:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574720.130973:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574720.130974:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574720.130976:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574720.130978:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516733184:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574720.130985:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574720.130986:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574720.130987:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574720.130989:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574720.130991:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574720.130992:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574720.130995:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574720.130996:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574720.130999:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574720.131001:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516733184, portal 4 00000100:00000001:0.0:1788574720.131003:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574720.131003:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788574720.131008:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516733184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.131012:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574720.131013:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574720.131015:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574720.131016:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516733184, offset 0 00000400:00000200:0.0:1788574720.131019:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574720.131021:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574720.131024:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574720.131026:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574720.131028:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574720.131029:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574720.131031:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574720.131033:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547380:547380:256:4294967295] 192.168.202.128@tcp LPNI seq info [547303:547303:8:4294967295] 00000400:00000200:0.0:1788574720.131036:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574720.131040:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574720.131043:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574720.131046:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574720.131048:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574720.131053:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788574720.131060:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574720.131062:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574720.131064:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574720.131066:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574720.131069:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574720.131070:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574720.131072:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574720.131074:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516733184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.131080:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574720.131082:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516733184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.131086:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.131087:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574720.131089:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574720.131091:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574720.131094:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574720.131097:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310080169 00000100:00000001:0.0:1788574720.131099:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310083821 00000100:00000001:0.0:1788574720.131102:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310080169 00000100:00000001:0.0:1788574720.131104:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310083821 00000100:00000001:0.0:1788574720.131106:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574720.131108:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.131110:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574720.131114:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574720/real 1788574720] req@ffff8fc4ac9f6a00 x1875440516733184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574720.131123:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574720.131124:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574720.131131:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574720.131134:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574720.131135:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574720.131139:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574720.131143:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516733184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574720.131151:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516733184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.131156:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574720.131157:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.131158:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574720.131159:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574720.131162:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516733184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.131166:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574720.131169:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516733184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.131173:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574720.131174:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.131176:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574720.131176:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.131178:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574720.131179:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574720.131181:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574720.131183:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574720.131184:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574720.131186:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516733184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574720.131191:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516733184:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574720.131197:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.131199:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574720.131201:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516733184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574720.131206:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574720.131207:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574720.131208:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574720.131210:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574720.131211:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574720.131212:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574720.131213:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574720.131214:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35800. 02000000:00000001:0.0:1788574720.131216:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574720.131217:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574720.131218:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788574720.131219:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574720.131220:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.131222:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.131224:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574720.131225:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.131226:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574720.131227:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574720.640914:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574720.640923:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574720.640926:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574720.640932:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574720.640939:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574720.640943:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.640958:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574720.640959:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574720.640961:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574721.345020:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574721.345024:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574721.345026:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574721.664980:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574721.664985:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574721.665004:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574721.665007:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574721.665009:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574721.665011:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574721.665013:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574721.665014:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574721.665016:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574722.688975:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574722.688979:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574722.688981:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574722.688984:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574722.688987:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574722.688990:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574722.688991:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574722.688993:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574722.688995:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574723.392982:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574723.392985:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574723.392988:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574723.712913:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574723.712921:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574723.712926:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574723.712933:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574723.712938:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574723.712944:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574723.712956:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574723.712960:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574723.712964:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574723.968918:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310083821, at 4310083840 00000100:00000010:0.0:1788574723.968934:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574724.416940:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574724.416943:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574724.416946:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574724.736903:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574724.736912:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574724.736916:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574724.736922:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574724.736938:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574724.736944:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574724.736953:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574724.736957:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574724.736960:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574725.249091:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788574725.760929:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574725.760936:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574725.760942:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574725.760947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574725.760959:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574725.760965:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574725.760971:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574725.760975:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574725.760980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574726.465040:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574726.465044:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574726.465046:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574726.784896:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574726.784902:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574726.784907:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574726.784927:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574726.784932:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574726.784936:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574726.784948:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574726.784952:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574726.784955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574726.784990:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574726.784993:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574726.784995:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574726.784997:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574726.785000:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574726.785002:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574726.785005:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574726.785007:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574726.785009:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574727.808969:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574727.808976:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574727.808992:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574727.808997:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574727.809004:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574727.809009:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574727.809031:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574727.809037:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574727.809043:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574728.513031:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574728.513036:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574728.513038:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574728.832892:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574728.832901:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574728.832907:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574728.832915:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574728.832924:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574728.832930:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574728.832954:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574728.832958:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574728.832963:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574729.088895:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310088941, at 4310088960 00000100:00000010:0.0:1788574729.088910:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfa80. 00000020:00000001:0.0:1788574729.537088:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574729.537094:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574729.537098:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574729.856985:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574729.856995:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574729.857004:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574729.857021:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574729.857034:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574729.857044:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574729.857064:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574729.857069:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574729.857076:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574730.880901:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574730.880910:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574730.880917:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574730.880924:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574730.880944:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574730.880949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574730.880964:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574730.880968:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574730.880971:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574731.584991:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574731.584996:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574731.585001:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574731.904901:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574731.904910:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574731.904919:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574731.904923:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574731.904932:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574731.904937:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.904961:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574731.904965:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574731.904969:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574732.928936:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574732.928944:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574732.928948:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574732.928956:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574732.928964:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574732.928970:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574732.928983:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574732.928987:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574732.928990:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574733.632946:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574733.632950:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574733.632954:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574733.952931:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574733.952938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574733.952954:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574733.952961:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574733.952969:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574733.952974:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574733.952979:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574733.952985:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574733.952993:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574734.208907:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310094061, at 4310094080 00000100:00000010:0.0:1788574734.208923:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf4e0. 00000020:00000001:0.0:1788574734.657010:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574734.657014:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574734.657016:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574734.976894:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574734.976904:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574734.976908:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574734.976916:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574734.976923:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574734.976931:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574734.976948:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574734.976950:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574734.976952:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.000886:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574736.000903:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574736.000910:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574736.000917:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574736.000925:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574736.000932:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574736.000937:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574736.000943:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574736.000950:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574736.000983:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574736.000988:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574736.000991:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574736.000994:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574736.000999:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574736.001002:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574736.001004:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574736.001006:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574736.001008:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574736.704998:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574736.705002:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574736.705005:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574737.024902:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574737.024907:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574737.024931:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574737.024936:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574737.024943:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574737.024949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574737.024965:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574737.024970:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574737.024973:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574738.048944:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574738.048951:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574738.048952:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574738.048962:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574738.048967:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574738.048970:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574738.048973:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574738.048977:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574738.048981:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574738.049038:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574738.049042:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574738.049045:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574738.049049:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574738.049055:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574738.049062:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574738.049076:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574738.049079:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574738.049083:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574738.753080:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574738.753087:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574738.753091:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574739.072912:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574739.072919:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574739.072923:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574739.072930:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574739.072938:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574739.072946:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574739.072962:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574739.072966:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574739.072971:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574739.072998:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574739.073009:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574739.073012:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574739.073015:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574739.073018:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574739.073022:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574739.073025:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574739.073027:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574739.073030:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574739.328909:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310099181, at 4310099200 00000100:00000010:0.0:1788574739.328933:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf1e0. 00000020:00000001:0.0:1788574739.777009:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574739.777012:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574739.777015:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574740.096976:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574740.096981:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574740.096984:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574740.096987:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574740.096990:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574740.096996:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.097003:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574740.097006:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.097011:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.609144:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574740.609149:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574740.609153:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574740.609163:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516735232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.609180:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574740.609185:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574740.609189:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574740.609194:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574740.609198:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574740.609203:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516735232:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574740.609211:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574740.609216:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574740.609228:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574740.609231:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574740.609238:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574740.609241:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574740.609249:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574740.609254:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574740.609262:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574740.609269:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516735232, portal 25 00000100:00000001:0.0:1788574740.609273:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574740.609275:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788574740.609282:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516735232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.609305:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574740.609308:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574740.609310:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574740.609312:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516735232, offset 0 00000400:00000200:0.0:1788574740.609318:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574740.609327:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574740.609334:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574740.609341:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574740.609348:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574740.609357:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574740.609364:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574740.609375:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547396:547396:256:4294967295] 192.168.202.128@tcp LPNI seq info [547319:547319:8:4294967295] 00000400:00000200:0.0:1788574740.609383:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574740.609405:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574740.609417:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574740.609424:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574740.609433:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574740.609437:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788574740.609459:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574740.609469:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574740.609476:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574740.609487:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574740.609493:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574740.609498:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574740.609504:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574740.609512:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516735232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.609523:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574740.609529:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516735232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.609542:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.609548:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574740.609555:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574740.609562:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574740.609567:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310104301 00000100:00000010:0.0:1788574740.609572:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574740.609578:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574740.609585:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310104301 00000100:00000010:0.0:1788574740.609590:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50de0. 00000100:00000001:0.0:1788574740.609595:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574740.609599:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574740.609601:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.609604:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574740.609610:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574740/real 1788574740] req@ffff8fc4901a4e00 x1875440516735232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574740.609622:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574740.609624:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574740.609628:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574740.609632:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574740.609635:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574740.609640:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574740.609651:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516735232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574740.609679:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516735232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.609685:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574740.609686:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.609688:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574740.609689:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574740.609694:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516735232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.609701:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574740.609704:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516735232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.609727:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574740.609730:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.609734:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574740.609735:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.609739:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574740.609742:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574740.609747:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574740.609752:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574740.609758:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574740.609763:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516735232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574740.609778:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516735232:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574740.609784:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.609787:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574740.609793:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516735232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.609836:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574740.609839:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574740.609843:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574740.609849:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574740.609852:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574740.609855:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574740.609858:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574740.609863:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788574740.609870:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574740.609872:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574740.609874:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788574740.609879:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574740.609882:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.609888:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.609895:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574740.609898:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.609901:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574740.609904:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574740.609907:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574740.609913:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516735360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.609924:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574740.609927:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574740.609932:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574740.609936:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574740.609942:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574740.609951:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516735360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574740.609958:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574740.609963:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574740.609967:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574740.609972:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574740.609978:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574740.609985:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574740.609993:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574740.610001:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574740.610007:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574740.610013:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516735360, portal 10 00000100:00000001:0.0:1788574740.610015:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574740.610017:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788574740.610027:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516735360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610057:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574740.610062:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574740.610067:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574740.610069:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516735360, offset 0 00000400:00000200:0.0:1788574740.610078:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574740.610083:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574740.610089:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574740.610095:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574740.610100:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574740.610109:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574740.610112:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574740.610118:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547397:547397:256:4294967295] 192.168.202.128@tcp LPNI seq info [547320:547320:8:4294967295] 00000400:00000200:0.0:1788574740.610122:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574740.610127:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574740.610132:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574740.610135:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574740.610142:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574740.610147:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788574740.610172:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574740.610174:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574740.610179:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574740.610187:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574740.610192:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574740.610195:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574740.610202:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574740.610209:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516735360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610224:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574740.610230:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516735360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610243:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.610245:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574740.610252:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574740.610257:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574740.610263:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574740.610266:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310100499 00000100:00000001:0.0:1788574740.610270:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310104301 00000100:00000001:0.0:1788574740.610276:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310100499 00000100:00000001:0.0:1788574740.610280:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310104301 00000100:00000001:0.0:1788574740.610284:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574740.610288:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.610294:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574740.610299:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574740/real 1788574740] req@ffff8fc4901a6300 x1875440516735360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574740.610316:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574740.610320:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574740.610323:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574740.610325:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574740.610328:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574740.610332:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574740.610337:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516735360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574740.610344:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516735360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610383:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574740.610388:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.610392:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574740.610396:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574740.610403:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516735360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610412:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574740.610420:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516735360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610433:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574740.610437:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.610442:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574740.610443:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.610445:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574740.610448:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574740.610453:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574740.610460:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574740.610465:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574740.610471:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516735360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574740.610479:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516735360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574740.610484:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574740.610487:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516735488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610491:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574740.610492:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574740.610499:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574740.610501:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574740.610503:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574740.610506:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516735488:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574740.610509:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574740.610511:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574740.610513:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574740.610514:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574740.610517:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574740.610519:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574740.610523:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574740.610525:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574740.610529:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574740.610532:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516735488, portal 10 00000100:00000001:0.0:1788574740.610534:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574740.610536:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788574740.610541:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516735488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610545:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574740.610548:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574740.610551:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574740.610553:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516735488, offset 0 00000400:00000200:0.0:1788574740.610556:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574740.610559:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574740.610561:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574740.610564:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574740.610566:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574740.610569:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574740.610570:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574740.610573:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547398:547398:256:4294967295] 192.168.202.128@tcp LPNI seq info [547321:547321:8:4294967295] 00000400:00000200:0.0:1788574740.610576:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574740.610580:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574740.610587:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574740.610589:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574740.610592:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574740.610594:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788574740.610602:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574740.610603:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574740.610605:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574740.610608:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574740.610610:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574740.610611:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574740.610613:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574740.610615:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516735488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610621:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574740.610623:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516735488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610627:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.610628:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574740.610630:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574740.610632:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574740.610635:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574740.610636:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310100506 00000100:00000001:0.0:1788574740.610638:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310104301 00000100:00000001:0.0:1788574740.610639:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310100506 00000100:00000001:0.0:1788574740.610641:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310104301 00000100:00000001:0.0:1788574740.610642:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574740.610643:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.610645:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574740.610647:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574740/real 1788574740] req@ffff8fc4901a7480 x1875440516735488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574740.610656:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574740.610657:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574740.610671:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574740.610674:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574740.610675:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574740.610677:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574740.610679:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516735488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574740.610685:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516735488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610689:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574740.610690:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.610692:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574740.610692:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574740.610695:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516735488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610699:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574740.610701:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516735488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610706:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574740.610707:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.610708:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574740.610709:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.610710:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574740.610712:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574740.610714:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574740.610716:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574740.610717:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574740.610724:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516735488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574740.610729:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516735488:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574740.610732:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.610733:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574740.610735:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516735360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610739:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574740.610740:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574740.610742:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574740.610744:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574740.610745:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574740.610746:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574740.610747:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574740.610748:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788574740.610750:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574740.610751:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574740.610752:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788574740.610755:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574740.610756:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.610757:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574740.610759:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516735488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610764:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574740.610764:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574740.610765:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574740.610767:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574740.610767:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574740.610768:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574740.610769:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574740.610770:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788574740.610771:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574740.610774:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574740.610775:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788574740.610778:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574740.610779:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.610780:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.610783:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574740.610784:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.610785:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574740.610786:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574740.610787:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574740.610790:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516735616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610794:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574740.610795:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574740.610797:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574740.610798:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574740.610800:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574740.610801:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516735616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574740.610803:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574740.610804:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574740.610805:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574740.610807:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574740.610823:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574740.610825:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574740.610828:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574740.610830:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574740.610832:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574740.610835:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516735616, portal 4 00000100:00000001:0.0:1788574740.610836:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574740.610837:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788574740.610841:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516735616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610846:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574740.610847:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574740.610852:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574740.610854:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516735616, offset 0 00000400:00000200:0.0:1788574740.610857:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574740.610860:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574740.610862:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574740.610864:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574740.610866:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574740.610868:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574740.610870:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574740.610872:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547399:547399:256:4294967295] 192.168.202.128@tcp LPNI seq info [547322:547322:8:4294967295] 00000400:00000200:0.0:1788574740.610874:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574740.610878:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574740.610881:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574740.610883:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574740.610886:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574740.610888:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788574740.610895:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574740.610897:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574740.610899:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574740.610901:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574740.610903:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574740.610904:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574740.610906:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574740.610908:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516735616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610913:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574740.610915:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516735616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610919:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.610920:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574740.610927:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574740.610929:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574740.610932:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574740.610934:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310100647 00000100:00000001:0.0:1788574740.610936:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310104301 00000100:00000001:0.0:1788574740.610939:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310100647 00000100:00000001:0.0:1788574740.610941:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310104301 00000100:00000001:0.0:1788574740.610943:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574740.610945:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.610947:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574740.610952:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574740/real 1788574740] req@ffff8fc4901a4700 x1875440516735616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574740.610960:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574740.610961:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574740.610963:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574740.610966:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574740.610968:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574740.610970:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574740.610974:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516735616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574740.610983:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516735616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.610990:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574740.610991:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.610993:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574740.610994:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574740.610996:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516735616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.611001:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574740.611003:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516735616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.611012:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574740.611014:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.611015:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574740.611016:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.611017:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574740.611019:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574740.611020:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574740.611022:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574740.611024:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574740.611026:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516735616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574740.611032:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516735616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574740.611034:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574740.611036:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516735744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.611039:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574740.611040:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574740.611042:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574740.611043:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574740.611044:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574740.611046:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516735744:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574740.611048:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574740.611050:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574740.611051:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574740.611052:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574740.611054:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574740.611055:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574740.611058:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574740.611062:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574740.611064:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574740.611067:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516735744, portal 4 00000100:00000001:0.0:1788574740.611068:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574740.611069:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788574740.611073:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516735744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.611077:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574740.611079:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574740.611080:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574740.611082:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516735744, offset 0 00000400:00000200:0.0:1788574740.611084:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574740.611087:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574740.611089:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574740.611091:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574740.611093:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574740.611095:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574740.611096:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574740.611099:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547400:547400:256:4294967295] 192.168.202.128@tcp LPNI seq info [547323:547323:8:4294967295] 00000400:00000200:0.0:1788574740.611101:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574740.611105:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574740.611108:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574740.611111:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574740.611113:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574740.611115:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788574740.611122:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574740.611123:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574740.611126:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574740.611128:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574740.611130:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574740.611131:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574740.611136:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574740.611139:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516735744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.611144:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574740.611146:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516735744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.611150:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.611151:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574740.611153:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574740.611155:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574740.611157:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574740.611158:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310100649 00000100:00000001:0.0:1788574740.611159:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310104301 00000100:00000001:0.0:1788574740.611161:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310100649 00000100:00000001:0.0:1788574740.611162:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310104301 00000100:00000001:0.0:1788574740.611164:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574740.611165:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.611166:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574740.611168:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574740/real 1788574740] req@ffff8fc4901a5880 x1875440516735744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574740.611173:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574740.611174:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574740.611175:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574740.611176:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574740.611177:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574740.611178:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574740.611180:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516735744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574740.611185:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516735744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.611196:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574740.611197:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.611198:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574740.611199:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574740.611202:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516735744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.611206:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574740.611208:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516735744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.611213:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574740.611214:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.611215:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574740.611216:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.611217:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574740.611219:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574740.611220:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574740.611222:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574740.611223:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574740.611225:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516735744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574740.611230:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516735744:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574740.611232:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.611233:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574740.611236:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516735616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.611240:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574740.611241:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574740.611245:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574740.611246:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574740.611248:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574740.611249:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574740.611250:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574740.611251:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788574740.611253:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574740.611254:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574740.611255:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788574740.611257:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574740.611258:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.611259:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574740.611262:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516735744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574740.611266:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574740.611267:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574740.611268:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574740.611269:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574740.611270:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574740.611271:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574740.611272:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574740.611272:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788574740.611274:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574740.611274:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574740.611275:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788574740.611277:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574740.611278:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.611279:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574740.611281:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574740.611282:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574740.611283:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574740.611284:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574741.120912:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574741.120921:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574741.120925:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574741.120937:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574741.120945:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574741.120964:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574741.120982:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574741.120985:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574741.120989:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574741.825035:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574741.825040:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574741.825052:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574742.144947:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574742.144953:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574742.144957:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574742.144962:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574742.144967:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574742.144971:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574742.144987:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574742.144991:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574742.144995:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574743.168945:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574743.168951:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574743.168956:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574743.168960:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574743.168968:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574743.168975:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574743.168981:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574743.168986:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574743.168990:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574743.873028:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574743.873033:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574743.873037:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574744.192970:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574744.192976:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574744.192980:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574744.192994:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574744.192998:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574744.193001:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574744.193003:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574744.193005:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574744.193008:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574744.448889:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310104301, at 4310104320 00000100:00000010:0.0:1788574744.448914:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574744.896990:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574744.896993:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574744.896996:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574745.216922:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574745.216926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574745.216931:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574745.216937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574745.216942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574745.216947:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.216963:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574745.216967:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.216970:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.729302:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574745.729307:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574745.729311:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574745.729320:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516735872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.729334:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574745.729337:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574745.729344:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574745.729347:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574745.729351:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574745.729357:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516735872:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574745.729363:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574745.729366:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574745.729369:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574745.729375:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574745.729381:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574745.729385:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574745.729393:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574745.729397:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574745.729406:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574745.729413:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516735872, portal 25 00000100:00000001:0.0:1788574745.729416:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574745.729425:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788574745.729432:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516735872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.729444:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574745.729450:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574745.729455:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574745.729458:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516735872, offset 0 00000400:00000200:0.0:1788574745.729466:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574745.729483:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574745.729491:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574745.729497:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574745.729504:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574745.729509:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574745.729512:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574745.729519:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547401:547401:256:4294967295] 192.168.202.128@tcp LPNI seq info [547324:547324:8:4294967295] 00000400:00000200:0.0:1788574745.729529:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574745.729542:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574745.729552:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574745.729558:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574745.729565:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574745.729570:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574745.729592:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574745.729600:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574745.729608:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574745.729614:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574745.729618:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574745.729621:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574745.729628:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574745.729633:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516735872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.729649:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574745.729661:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516735872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.729674:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.729678:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574745.729684:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574745.729697:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574745.729700:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310109421 00000100:00000010:0.0:1788574745.729706:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574745.729713:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574745.729720:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310109421 00000100:00000010:0.0:1788574745.729724:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50e40. 00000100:00000001:0.0:1788574745.729731:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574745.729737:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574745.729744:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.729750:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574745.729760:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574745/real 1788574745] req@ffff8fc4ac822d80 x1875440516735872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574745.729777:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574745.729781:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574745.729784:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574745.729789:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574745.729791:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574745.729793:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574745.729798:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516735872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574745.729870:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516735872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.729889:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574745.729893:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.729898:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574745.729902:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574745.729908:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516735872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.729926:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574745.729934:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516735872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.729949:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574745.729952:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.729955:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574745.729958:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.729961:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574745.729963:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574745.729967:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574745.729973:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574745.729977:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574745.729983:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516735872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574745.729995:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516735872:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574745.730000:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730001:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574745.730005:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516735872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730015:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574745.730018:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574745.730021:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574745.730025:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574745.730029:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574745.730032:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574745.730035:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574745.730039:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788574745.730043:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574745.730049:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574745.730052:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788574745.730057:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574745.730059:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730065:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730070:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574745.730073:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.730078:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574745.730080:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574745.730083:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574745.730088:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516736000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730097:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574745.730099:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574745.730101:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574745.730103:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574745.730108:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574745.730111:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736000:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574745.730116:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574745.730118:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574745.730120:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574745.730122:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574745.730126:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574745.730130:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574745.730137:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574745.730144:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574745.730150:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574745.730157:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516736000, portal 10 00000100:00000001:0.0:1788574745.730162:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574745.730165:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788574745.730177:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516736000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730195:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574745.730211:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574745.730218:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574745.730221:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516736000, offset 0 00000400:00000200:0.0:1788574745.730228:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574745.730240:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574745.730244:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574745.730249:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574745.730254:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574745.730261:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574745.730269:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574745.730279:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547402:547402:256:4294967295] 192.168.202.128@tcp LPNI seq info [547325:547325:8:4294967295] 00000400:00000200:0.0:1788574745.730291:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574745.730307:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574745.730320:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574745.730327:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574745.730338:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574745.730344:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574745.730369:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574745.730373:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574745.730379:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574745.730386:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574745.730392:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574745.730396:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574745.730400:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574745.730406:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516736000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730421:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574745.730428:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516736000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730437:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.730440:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574745.730450:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574745.730454:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574745.730461:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574745.730466:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310105619 00000100:00000001:0.0:1788574745.730470:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310109421 00000100:00000001:0.0:1788574745.730473:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310105619 00000100:00000001:0.0:1788574745.730477:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310109421 00000100:00000001:0.0:1788574745.730481:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574745.730485:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.730487:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574745.730493:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574745/real 1788574745] req@ffff8fc4ac821f80 x1875440516736000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574745.730508:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574745.730510:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574745.730512:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574745.730515:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574745.730518:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574745.730521:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574745.730523:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516736000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574745.730530:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516736000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730534:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574745.730535:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730537:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574745.730538:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574745.730540:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516736000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730544:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574745.730547:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516736000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730557:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574745.730559:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730560:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574745.730561:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730562:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574745.730563:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574745.730565:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574745.730567:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574745.730568:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574745.730571:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516736000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574745.730576:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736000:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574745.730579:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574745.730581:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516736128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730585:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574745.730586:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574745.730587:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574745.730588:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574745.730589:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574745.730591:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736128:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574745.730593:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574745.730594:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574745.730596:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574745.730597:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574745.730599:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574745.730600:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574745.730606:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574745.730608:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574745.730610:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574745.730612:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516736128, portal 10 00000100:00000001:0.0:1788574745.730613:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574745.730614:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788574745.730618:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516736128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730623:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574745.730624:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574745.730626:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574745.730627:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516736128, offset 0 00000400:00000200:0.0:1788574745.730630:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574745.730638:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574745.730640:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574745.730642:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574745.730644:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574745.730646:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574745.730648:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574745.730650:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547403:547403:256:4294967295] 192.168.202.128@tcp LPNI seq info [547326:547326:8:4294967295] 00000400:00000200:0.0:1788574745.730653:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574745.730657:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574745.730660:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574745.730662:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574745.730665:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574745.730667:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574745.730674:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574745.730676:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574745.730678:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574745.730680:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574745.730682:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574745.730686:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574745.730689:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574745.730691:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516736128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730697:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574745.730699:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516736128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730703:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.730704:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574745.730706:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574745.730709:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574745.730711:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574745.730712:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310105626 00000100:00000001:0.0:1788574745.730714:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310109421 00000100:00000001:0.0:1788574745.730716:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310105626 00000100:00000001:0.0:1788574745.730717:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310109421 00000100:00000001:0.0:1788574745.730719:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574745.730720:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.730721:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574745.730725:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574745/real 1788574745] req@ffff8fc4ac823480 x1875440516736128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574745.730733:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574745.730734:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574745.730736:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574745.730738:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574745.730740:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574745.730742:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574745.730745:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516736128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574745.730754:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516736128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730767:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574745.730769:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730771:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574745.730772:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574745.730777:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516736128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730783:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574745.730786:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516736128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730790:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574745.730792:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730793:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574745.730794:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730795:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574745.730797:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574745.730798:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574745.730801:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574745.730803:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574745.730826:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516736128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574745.730836:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736128:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574745.730840:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730842:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574745.730846:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516736000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730852:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574745.730857:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574745.730859:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574745.730861:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574745.730862:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574745.730863:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574745.730865:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574745.730866:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788574745.730867:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574745.730868:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574745.730870:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788574745.730871:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574745.730872:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730873:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574745.730876:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516736128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730880:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574745.730881:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574745.730882:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574745.730883:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574745.730884:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574745.730885:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574745.730886:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574745.730887:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788574745.730888:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574745.730888:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574745.730889:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788574745.730890:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574745.730891:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730892:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.730895:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574745.730896:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.730897:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574745.730898:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574745.730899:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574745.730901:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516736256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730905:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574745.730909:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574745.730911:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574745.730912:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574745.730913:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574745.730915:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736256:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574745.730917:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574745.730919:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574745.730920:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574745.730921:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574745.730923:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574745.730924:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574745.730927:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574745.730929:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574745.730932:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574745.730934:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516736256, portal 4 00000100:00000001:0.0:1788574745.730935:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574745.730936:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788574745.730940:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516736256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.730944:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574745.730946:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574745.730947:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574745.730949:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516736256, offset 0 00000400:00000200:0.0:1788574745.730951:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574745.730954:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574745.730956:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574745.730958:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574745.730960:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574745.730961:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574745.730963:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574745.730965:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547404:547404:256:4294967295] 192.168.202.128@tcp LPNI seq info [547327:547327:8:4294967295] 00000400:00000200:0.0:1788574745.730968:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574745.730972:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574745.730979:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574745.730981:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574745.730983:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574745.730985:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574745.730992:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574745.730994:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574745.730996:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574745.730998:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574745.731001:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574745.731002:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574745.731003:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574745.731006:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516736256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731011:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574745.731013:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516736256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731017:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.731018:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574745.731020:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574745.731022:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574745.731025:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574745.731026:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310105767 00000100:00000001:0.0:1788574745.731028:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310109421 00000100:00000001:0.0:1788574745.731029:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310105767 00000100:00000001:0.0:1788574745.731031:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310109421 00000100:00000001:0.0:1788574745.731032:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574745.731033:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.731034:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574745.731037:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574745/real 1788574745] req@ffff8fc4ac822300 x1875440516736256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574745.731047:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574745.731048:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574745.731049:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574745.731050:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574745.731051:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574745.731053:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574745.731055:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516736256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574745.731060:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516736256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731064:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574745.731065:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.731066:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574745.731067:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574745.731069:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516736256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731073:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574745.731076:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516736256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731080:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574745.731082:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.731083:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574745.731084:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.731085:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574745.731086:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574745.731088:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574745.731090:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574745.731091:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574745.731096:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516736256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574745.731102:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736256:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574745.731104:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574745.731106:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516736384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731110:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574745.731111:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574745.731112:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574745.731113:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574745.731114:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574745.731116:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736384:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574745.731118:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574745.731119:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574745.731120:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574745.731121:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574745.731123:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574745.731124:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574745.731127:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574745.731128:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574745.731130:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574745.731132:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516736384, portal 4 00000100:00000001:0.0:1788574745.731134:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574745.731135:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788574745.731139:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516736384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731145:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574745.731148:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574745.731150:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574745.731153:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516736384, offset 0 00000400:00000200:0.0:1788574745.731162:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574745.731166:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574745.731170:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574745.731173:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574745.731175:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574745.731178:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574745.731181:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574745.731184:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547405:547405:256:4294967295] 192.168.202.128@tcp LPNI seq info [547328:547328:8:4294967295] 00000400:00000200:0.0:1788574745.731188:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574745.731208:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574745.731213:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574745.731216:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574745.731219:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574745.731220:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788574745.731228:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574745.731229:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574745.731231:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574745.731233:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574745.731236:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574745.731237:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574745.731238:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574745.731241:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516736384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731246:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574745.731248:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516736384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731253:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.731254:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574745.731256:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574745.731258:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574745.731260:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574745.731266:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310105769 00000100:00000001:0.0:1788574745.731268:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310109421 00000100:00000001:0.0:1788574745.731269:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310105769 00000100:00000001:0.0:1788574745.731270:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310109421 00000100:00000001:0.0:1788574745.731272:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574745.731273:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.731274:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574745.731277:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574745/real 1788574745] req@ffff8fc4ac820700 x1875440516736384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574745.731282:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574745.731283:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574745.731284:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574745.731286:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574745.731287:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574745.731288:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574745.731290:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516736384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574745.731296:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516736384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731300:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574745.731301:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.731302:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574745.731303:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574745.731305:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516736384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731309:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574745.731311:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516736384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731315:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574745.731319:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.731321:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574745.731321:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.731323:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574745.731324:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574745.731326:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574745.731328:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574745.731329:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574745.731332:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516736384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574745.731337:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736384:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574745.731339:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.731341:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574745.731342:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516736256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731353:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574745.731354:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574745.731355:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574745.731360:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574745.731361:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574745.731362:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574745.731364:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574745.731365:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788574745.731366:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574745.731367:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574745.731368:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788574745.731370:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574745.731371:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.731372:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574745.731374:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516736384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574745.731385:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574745.731386:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574745.731388:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574745.731389:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574745.731390:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574745.731391:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574745.731392:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574745.731393:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788574745.731395:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574745.731395:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574745.731396:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788574745.731397:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574745.731398:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.731399:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574745.731402:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574745.731403:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574745.731404:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574745.731405:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574746.241101:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574746.241113:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574746.241124:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574746.241139:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574746.241145:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574746.241166:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574746.241170:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574746.241172:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574746.241176:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574746.945020:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574746.945024:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574746.945028:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574747.264920:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574747.264927:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574747.264931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574747.264937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574747.264942:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574747.264947:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574747.264968:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574747.264971:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574747.264974:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574748.288934:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574748.288939:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574748.288942:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574748.288945:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574748.288950:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574748.288956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.288961:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574748.288963:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574748.288990:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574748.992996:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574748.993000:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574748.993003:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574749.312918:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574749.312928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574749.312932:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574749.312940:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574749.312946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574749.312949:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574749.312952:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574749.312957:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574749.312960:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574749.312995:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574749.312998:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574749.313001:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574749.313004:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574749.313009:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574749.313012:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574749.313016:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574749.313018:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574749.313021:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574749.568929:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310109421, at 4310109440 00000100:00000010:0.0:1788574749.568944:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574750.017014:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574750.017019:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574750.017039:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574750.336952:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574750.336961:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574750.336966:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574750.336975:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574750.336984:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574750.336989:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.337005:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574750.337009:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.337012:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.337030:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574750.337035:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574750.337039:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574750.337042:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574750.337048:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574750.337053:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.337056:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574750.337058:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.337062:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.849169:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574750.849177:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574750.849183:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574750.849195:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516736512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.849217:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574750.849221:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574750.849230:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574750.849235:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574750.849242:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574750.849250:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516736512:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574750.849263:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574750.849270:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574750.849274:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574750.849281:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574750.849289:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574750.849296:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574750.849317:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574750.849326:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574750.849339:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574750.849349:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516736512, portal 25 00000100:00000001:0.0:1788574750.849357:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574750.849363:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788574750.849376:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516736512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.849399:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574750.849405:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574750.849413:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574750.849419:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516736512, offset 0 00000400:00000200:0.0:1788574750.849434:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574750.849458:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574750.849471:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574750.849483:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574750.849493:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574750.849502:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574750.849511:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574750.849523:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547406:547406:256:4294967295] 192.168.202.128@tcp LPNI seq info [547329:547329:8:4294967295] 00000400:00000200:0.0:1788574750.849537:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574750.849556:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574750.849568:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574750.849578:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574750.849591:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574750.849597:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788574750.849629:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574750.849635:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574750.849645:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574750.849657:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574750.849664:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574750.849675:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574750.849686:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574750.849698:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516736512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.849721:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574750.849730:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516736512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.849749:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.849755:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574750.849765:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574750.849775:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574750.849783:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310114541 00000100:00000010:0.0:1788574750.849793:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574750.849803:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574750.849852:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310114542 00000100:00000010:0.0:1788574750.849877:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50a80. 00000100:00000001:0.0:1788574750.849886:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574750.849894:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574750.849902:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.849916:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574750.849937:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574750/real 1788574750] req@ffff8fc4ac821c00 x1875440516736512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574750.849948:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574750.849950:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574750.849953:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574750.849956:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574750.849958:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574750.849961:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574750.849965:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516736512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574750.849973:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516736512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.849988:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574750.849990:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.849993:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574750.849995:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574750.850001:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516736512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850018:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574750.850023:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516736512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850031:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574750.850035:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.850036:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574750.850038:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.850040:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574750.850042:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574750.850046:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574750.850050:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574750.850052:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574750.850056:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516736512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574750.850064:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516736512:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574750.850069:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.850071:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574750.850074:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516736512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850080:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574750.850083:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574750.850090:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574750.850094:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574750.850096:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574750.850098:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574750.850100:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574750.850103:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574750.850107:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574750.850109:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574750.850111:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788574750.850114:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574750.850115:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.850117:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.850124:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574750.850125:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.850127:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574750.850129:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574750.850131:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574750.850135:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516736640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850143:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574750.850144:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574750.850146:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574750.850148:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574750.850151:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574750.850154:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736640:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574750.850157:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574750.850160:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574750.850162:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574750.850164:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574750.850167:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574750.850169:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574750.850173:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574750.850177:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574750.850180:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574750.850184:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516736640, portal 10 00000100:00000001:0.0:1788574750.850193:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574750.850194:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788574750.850201:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516736640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850208:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574750.850210:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574750.850213:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574750.850216:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516736640, offset 0 00000400:00000200:0.0:1788574750.850220:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574750.850224:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574750.850228:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574750.850231:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574750.850234:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574750.850237:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574750.850239:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574750.850242:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547407:547407:256:4294967295] 192.168.202.128@tcp LPNI seq info [547330:547330:8:4294967295] 00000400:00000200:0.0:1788574750.850246:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574750.850253:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574750.850257:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574750.850261:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574750.850264:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574750.850268:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788574750.850280:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574750.850282:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574750.850285:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574750.850289:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574750.850292:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574750.850294:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574750.850297:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574750.850301:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516736640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850321:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574750.850325:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516736640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850331:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.850333:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574750.850337:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574750.850341:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574750.850344:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574750.850347:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310110739 00000100:00000001:0.0:1788574750.850349:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310114541 00000100:00000001:0.0:1788574750.850352:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310110739 00000100:00000001:0.0:1788574750.850354:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310114542 00000100:00000001:0.0:1788574750.850356:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574750.850359:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.850361:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574750.850365:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574750/real 1788574750] req@ffff8fc4ac820a80 x1875440516736640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574750.850374:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574750.850375:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574750.850377:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574750.850380:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574750.850382:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574750.850384:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574750.850387:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516736640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574750.850396:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516736640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850403:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574750.850411:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.850413:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574750.850414:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574750.850419:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516736640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850427:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574750.850466:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516736640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850474:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574750.850477:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.850479:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574750.850480:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.850482:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574750.850484:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574750.850487:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574750.850490:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574750.850492:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574750.850497:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516736640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574750.850505:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736640:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574750.850508:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574750.850511:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516736768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850567:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574750.850570:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574750.850572:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574750.850574:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574750.850576:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574750.850587:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736768:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574750.850590:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574750.850592:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574750.850594:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574750.850595:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574750.850599:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574750.850600:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574750.850604:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574750.850607:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574750.850610:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574750.850613:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516736768, portal 10 00000100:00000001:0.0:1788574750.850615:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574750.850616:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788574750.850622:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516736768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850629:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574750.850631:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574750.850634:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574750.850637:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516736768, offset 0 00000400:00000200:0.0:1788574750.850642:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574750.850648:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574750.850653:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574750.850657:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574750.850660:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574750.850662:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574750.850665:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574750.850669:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547408:547408:256:4294967295] 192.168.202.128@tcp LPNI seq info [547331:547331:8:4294967295] 00000400:00000200:0.0:1788574750.850673:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574750.850679:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574750.850683:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574750.850687:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574750.850696:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574750.850699:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788574750.850711:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574750.850713:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574750.850717:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574750.850721:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574750.850724:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574750.850726:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574750.850729:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574750.850735:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516736768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850744:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574750.850749:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516736768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850756:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.850758:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574750.850762:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574750.850766:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574750.850770:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574750.850772:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310110746 00000100:00000001:0.0:1788574750.850775:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310114541 00000100:00000001:0.0:1788574750.850784:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310110746 00000100:00000001:0.0:1788574750.850787:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310114542 00000100:00000001:0.0:1788574750.850789:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574750.850791:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.850793:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574750.850797:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574750/real 1788574750] req@ffff8fc4ac823800 x1875440516736768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574750.850806:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574750.850833:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574750.850835:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574750.850838:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574750.850840:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574750.850843:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574750.850847:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516736768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574750.850857:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516736768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850879:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574750.850881:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.850883:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574750.850885:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574750.850890:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516736768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850897:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574750.850901:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516736768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850908:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574750.850911:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.850913:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574750.850914:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.850916:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574750.850919:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574750.850923:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574750.850926:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574750.850928:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574750.850932:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516736768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574750.850952:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736768:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574750.850956:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574750.850960:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516736896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.850966:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574750.850968:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574750.850970:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574750.850972:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574750.850974:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574750.850977:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736896:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574750.850981:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574750.850984:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574750.850985:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574750.850987:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574750.850991:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574750.850993:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574750.850997:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574750.851000:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574750.851003:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574750.851050:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516736896, portal 4 00000100:00000001:0.0:1788574750.851054:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574750.851055:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788574750.851063:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516736896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851071:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574750.851074:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574750.851077:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574750.851080:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516736896, offset 0 00000400:00000200:0.0:1788574750.851085:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574750.851096:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574750.851101:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574750.851104:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574750.851107:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574750.851110:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574750.851113:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574750.851116:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547409:547409:256:4294967295] 192.168.202.128@tcp LPNI seq info [547332:547332:8:4294967295] 00000400:00000200:0.0:1788574750.851120:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574750.851128:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574750.851132:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574750.851136:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574750.851162:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574750.851166:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788574750.851184:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574750.851187:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574750.851190:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574750.851194:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574750.851198:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574750.851199:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574750.851202:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574750.851206:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516736896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851215:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574750.851218:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516736896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851226:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.851227:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574750.851231:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574750.851234:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574750.851237:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574750.851245:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310110887 00000100:00000001:0.0:1788574750.851248:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310114541 00000100:00000001:0.0:1788574750.851250:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310110887 00000100:00000001:0.0:1788574750.851252:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310114542 00000100:00000001:0.0:1788574750.851254:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574750.851256:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.851258:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574750.851263:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574750/real 1788574750] req@ffff8fc4ac820e00 x1875440516736896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574750.851271:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574750.851273:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574750.851274:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574750.851277:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574750.851279:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574750.851282:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574750.851286:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516736896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574750.851294:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516736896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851301:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574750.851302:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.851305:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574750.851306:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574750.851310:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516736896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851317:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574750.851321:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516736896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851328:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574750.851337:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.851338:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574750.851340:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.851342:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574750.851344:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574750.851347:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574750.851351:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574750.851353:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574750.851358:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516736896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574750.851367:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516736896:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574750.851371:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574750.851374:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516737024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851381:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574750.851383:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574750.851386:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574750.851387:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574750.851389:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574750.851392:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516737024:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574750.851395:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574750.851398:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574750.851400:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574750.851402:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574750.851405:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574750.851407:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574750.851411:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574750.851415:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574750.851418:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574750.851421:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516737024, portal 4 00000100:00000001:0.0:1788574750.851428:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574750.851430:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788574750.851436:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516737024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851443:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574750.851445:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574750.851448:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574750.851451:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516737024, offset 0 00000400:00000200:0.0:1788574750.851455:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574750.851460:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574750.851463:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574750.851466:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574750.851470:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574750.851472:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574750.851475:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574750.851478:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547410:547410:256:4294967295] 192.168.202.128@tcp LPNI seq info [547333:547333:8:4294967295] 00000400:00000200:0.0:1788574750.851482:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574750.851488:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574750.851493:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574750.851496:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574750.851500:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574750.851503:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788574750.851515:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574750.851517:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574750.851520:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574750.851523:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574750.851526:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574750.851528:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574750.851531:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574750.851534:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516737024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851553:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574750.851557:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516737024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574775 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851564:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.851566:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574750.851570:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574750.851572:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574750.851577:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574750.851579:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310110889 00000100:00000001:0.0:1788574750.851581:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310114541 00000100:00000001:0.0:1788574750.851583:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310110889 00000100:00000001:0.0:1788574750.851586:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310114542 00000100:00000001:0.0:1788574750.851588:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574750.851590:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.851592:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574750.851596:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574750/real 1788574750] req@ffff8fc4ac821880 x1875440516737024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574750.851604:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574750.851606:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574750.851607:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574750.851610:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574750.851611:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574750.851613:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574750.851617:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516737024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574750.851625:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516737024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851632:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574750.851640:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.851642:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574750.851643:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574750.851648:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516737024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851655:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574750.851659:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516737024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851666:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574750.851669:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.851670:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574750.851671:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.851673:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574750.851675:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574750.851678:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574750.851681:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574750.851683:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574750.851687:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516737024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574750.851695:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516737024:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574750.851700:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.851702:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574750.851705:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516736640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851712:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574750.851714:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574750.851716:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574750.851720:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574750.851721:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574750.851730:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574750.851732:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574750.851734:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788574750.851737:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574750.851739:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574750.851740:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788574750.851744:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574750.851746:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.851747:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574750.851751:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516736768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851757:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574750.851759:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574750.851760:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574750.851762:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574750.851763:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574750.851765:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574750.851768:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574750.851769:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788574750.851771:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574750.851773:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574750.851774:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788574750.851777:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574750.851778:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.851780:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574750.851783:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516736896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851790:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574750.851791:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574750.851792:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574750.851795:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574750.851796:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574750.851797:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574750.851799:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574750.851800:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788574750.851803:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574750.851827:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574750.851829:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788574750.851833:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574750.851834:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.851836:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574750.851841:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516737024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574775 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574750.851848:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574750.851849:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574750.851851:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574750.851855:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574750.851857:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574750.851858:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574750.851860:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574750.851876:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788574750.851879:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574750.851880:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574750.851881:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788574750.851884:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574750.851885:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.851887:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574750.851891:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574750.851893:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574750.851895:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574750.851896:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574751.360948:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574751.360954:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574751.360957:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574751.360965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574751.360970:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574751.360977:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574751.361000:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574751.361006:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574751.361009:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574752.065015:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574752.065019:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574752.065021:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574752.384922:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574752.384928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574752.384933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574752.384944:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574752.384952:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574752.384958:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574752.384965:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574752.384967:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574752.384975:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574753.408922:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574753.408929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574753.408939:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574753.408946:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574753.408954:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574753.408959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574753.408967:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574753.408971:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574753.408974:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574754.112963:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574754.112967:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574754.112970:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574754.432901:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574754.432909:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574754.432913:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574754.432921:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574754.432928:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574754.432934:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574754.432941:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574754.432944:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574754.432948:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574754.432995:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574754.432999:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574754.433000:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574754.433002:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574754.433005:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574754.433006:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574754.433008:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574754.433010:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574754.433021:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574754.688875:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310114541, at 4310114560 00000100:00000010:0.0:1788574754.688889:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574755.137008:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574755.137013:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574755.137017:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574755.457013:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574755.457017:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574755.457018:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574755.457022:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574755.457024:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574755.457026:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.457028:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574755.457029:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.457031:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.969205:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574755.969210:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574755.969214:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574755.969223:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516737152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.969239:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574755.969242:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574755.969246:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574755.969250:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574755.969254:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574755.969261:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516737152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574755.969268:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574755.969272:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574755.969277:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574755.969282:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574755.969287:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574755.969291:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574755.969300:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574755.969303:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574755.969313:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574755.969329:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516737152, portal 25 00000100:00000001:0.0:1788574755.969335:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574755.969339:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788574755.969351:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516737152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.969370:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574755.969375:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574755.969379:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574755.969384:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516737152, offset 0 00000400:00000200:0.0:1788574755.969391:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574755.969402:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574755.969409:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574755.969415:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574755.969422:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574755.969430:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574755.969438:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574755.969446:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547411:547411:256:4294967295] 192.168.202.128@tcp LPNI seq info [547334:547334:8:4294967295] 00000400:00000200:0.0:1788574755.969457:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574755.969487:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574755.969500:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574755.969506:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574755.969515:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574755.969520:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574755.969542:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574755.969544:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574755.969549:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574755.969557:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574755.969562:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574755.969566:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574755.969574:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574755.969581:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516737152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.969606:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574755.969614:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516737152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.969630:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.969635:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574755.969641:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574755.969649:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574755.969653:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310119661 00000100:00000010:0.0:1788574755.969662:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574755.969670:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574755.969677:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310119661 00000100:00000010:0.0:1788574755.969682:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50720. 00000100:00000001:0.0:1788574755.969686:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574755.969691:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574755.969694:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.969700:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574755.969723:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574755/real 1788574755] req@ffff8fc4ac821880 x1875440516737152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574755.969736:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574755.969739:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574755.969741:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574755.969743:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574755.969745:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574755.969750:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574755.969754:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516737152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574755.969766:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516737152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.969792:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574755.969800:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.969804:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574755.969838:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574755.969845:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516737152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.969861:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574755.969868:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516737152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.969878:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574755.969883:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.969886:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574755.969888:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.969890:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574755.969891:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574755.969893:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574755.969900:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574755.969903:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574755.969910:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516737152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574755.969924:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516737152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574755.969932:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.969935:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574755.969941:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516737152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.969954:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574755.969958:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574755.969961:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574755.969968:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574755.969970:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574755.969979:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574755.969983:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574755.969987:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788574755.969991:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574755.969992:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574755.969995:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788574755.970000:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574755.970003:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.970007:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.970015:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574755.970019:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.970022:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574755.970024:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574755.970026:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574755.970032:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516737280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970043:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574755.970045:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574755.970049:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574755.970053:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574755.970056:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574755.970060:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516737280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574755.970067:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574755.970070:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574755.970073:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574755.970076:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574755.970079:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574755.970084:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574755.970089:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574755.970095:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574755.970099:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574755.970103:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516737280, portal 10 00000100:00000001:0.0:1788574755.970107:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574755.970111:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788574755.970123:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516737280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970142:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574755.970151:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574755.970157:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574755.970164:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516737280, offset 0 00000400:00000200:0.0:1788574755.970171:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574755.970179:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574755.970185:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574755.970190:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574755.970194:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574755.970199:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574755.970202:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574755.970209:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547412:547412:256:4294967295] 192.168.202.128@tcp LPNI seq info [547335:547335:8:4294967295] 00000400:00000200:0.0:1788574755.970214:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574755.970221:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574755.970227:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574755.970231:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574755.970237:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574755.970241:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574755.970259:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574755.970264:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574755.970267:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574755.970273:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574755.970277:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574755.970281:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574755.970286:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574755.970292:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516737280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970306:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574755.970314:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516737280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970355:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.970361:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574755.970368:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574755.970375:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574755.970381:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574755.970385:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310115859 00000100:00000001:0.0:1788574755.970389:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310119661 00000100:00000001:0.0:1788574755.970395:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310115859 00000100:00000001:0.0:1788574755.970400:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310119661 00000100:00000001:0.0:1788574755.970402:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574755.970408:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.970410:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574755.970416:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574755/real 1788574755] req@ffff8fc4ac820e00 x1875440516737280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574755.970430:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574755.970434:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574755.970438:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574755.970444:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574755.970449:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574755.970456:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574755.970463:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516737280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574755.970481:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516737280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970500:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574755.970503:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.970506:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574755.970510:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574755.970514:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516737280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970530:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574755.970539:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516737280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970548:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574755.970553:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.970557:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574755.970559:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.970561:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574755.970563:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574755.970565:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574755.970567:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574755.970569:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574755.970573:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516737280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574755.970579:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516737280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574755.970582:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574755.970584:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516737408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970589:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574755.970590:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574755.970591:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574755.970593:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574755.970594:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574755.970596:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516737408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574755.970598:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574755.970599:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574755.970603:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574755.970605:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574755.970607:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574755.970609:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574755.970612:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574755.970614:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574755.970616:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574755.970618:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516737408, portal 10 00000100:00000001:0.0:1788574755.970619:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574755.970621:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788574755.970625:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516737408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970629:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574755.970631:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574755.970632:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574755.970634:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516737408, offset 0 00000400:00000200:0.0:1788574755.970636:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574755.970639:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574755.970641:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574755.970643:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574755.970645:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574755.970646:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574755.970648:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574755.970650:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547413:547413:256:4294967295] 192.168.202.128@tcp LPNI seq info [547336:547336:8:4294967295] 00000400:00000200:0.0:1788574755.970653:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574755.970658:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574755.970661:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574755.970663:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574755.970666:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574755.970668:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574755.970674:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574755.970679:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574755.970681:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574755.970683:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574755.970686:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574755.970687:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574755.970689:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574755.970691:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516737408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970697:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574755.970699:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516737408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970703:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.970704:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574755.970722:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574755.970725:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574755.970727:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574755.970729:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310115866 00000100:00000001:0.0:1788574755.970730:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310119661 00000100:00000001:0.0:1788574755.970732:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310115866 00000100:00000001:0.0:1788574755.970733:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310119661 00000100:00000001:0.0:1788574755.970735:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574755.970736:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.970737:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574755.970740:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574755/real 1788574755] req@ffff8fc4ac823800 x1875440516737408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574755.970745:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574755.970746:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574755.970747:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574755.970749:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574755.970750:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574755.970756:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574755.970759:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516737408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574755.970764:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516737408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970768:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574755.970769:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.970770:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574755.970771:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574755.970774:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516737408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970778:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574755.970780:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516737408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970784:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574755.970786:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.970787:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574755.970788:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.970789:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574755.970791:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574755.970793:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574755.970795:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574755.970796:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574755.970798:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516737408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574755.970804:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516737408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574755.970806:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.970826:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574755.970829:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516737280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970834:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574755.970835:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574755.970837:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574755.970839:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574755.970840:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574755.970841:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574755.970842:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574755.970843:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788574755.970845:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574755.970846:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574755.970847:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788574755.970849:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574755.970850:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.970851:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574755.970853:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516737408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970857:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574755.970858:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574755.970858:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574755.970860:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574755.970860:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574755.970861:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574755.970862:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574755.970863:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788574755.970864:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574755.970865:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574755.970866:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788574755.970867:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574755.970868:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.970869:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.970871:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574755.970872:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.970874:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574755.970878:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574755.970879:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574755.970881:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516737536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970886:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574755.970887:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574755.970889:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574755.970891:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574755.970892:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574755.970894:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516737536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574755.970896:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574755.970897:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574755.970898:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574755.970900:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574755.970901:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574755.970903:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574755.970906:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574755.970907:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574755.970910:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574755.970912:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516737536, portal 4 00000100:00000001:0.0:1788574755.970914:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574755.970915:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788574755.970919:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516737536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970923:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574755.970925:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574755.970926:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574755.970927:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516737536, offset 0 00000400:00000200:0.0:1788574755.970930:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574755.970932:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574755.970935:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574755.970937:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574755.970939:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574755.970943:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574755.970945:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574755.970947:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547414:547414:256:4294967295] 192.168.202.128@tcp LPNI seq info [547337:547337:8:4294967295] 00000400:00000200:0.0:1788574755.970950:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574755.970955:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574755.970958:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574755.970960:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574755.970962:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574755.970964:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574755.970971:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574755.970972:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574755.970974:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574755.970976:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574755.970978:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574755.970980:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574755.970981:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574755.970984:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516737536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970989:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574755.970991:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516737536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.970995:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.970997:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574755.970999:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574755.971001:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574755.971003:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574755.971004:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310116007 00000100:00000001:0.0:1788574755.971006:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310119661 00000100:00000001:0.0:1788574755.971007:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310116007 00000100:00000001:0.0:1788574755.971012:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310119661 00000100:00000001:0.0:1788574755.971014:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574755.971016:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.971017:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574755.971020:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574755/real 1788574755] req@ffff8fc4ac820a80 x1875440516737536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574755.971025:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574755.971026:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574755.971027:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574755.971029:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574755.971030:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574755.971031:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574755.971033:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516737536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574755.971038:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516737536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.971042:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574755.971043:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.971044:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574755.971045:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574755.971047:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516737536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.971051:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574755.971054:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516737536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.971058:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574755.971060:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.971061:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574755.971062:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.971066:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574755.971068:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574755.971069:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574755.971072:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574755.971073:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574755.971076:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516737536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574755.971081:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516737536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574755.971083:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574755.971085:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516737664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.971089:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574755.971090:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574755.971091:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574755.971092:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574755.971093:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574755.971095:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516737664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574755.971097:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574755.971098:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574755.971099:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574755.971100:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574755.971101:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574755.971103:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574755.971105:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574755.971107:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574755.971110:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574755.971113:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516737664, portal 4 00000100:00000001:0.0:1788574755.971115:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574755.971116:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788574755.971122:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516737664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.971134:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574755.971137:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574755.971139:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574755.971143:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516737664, offset 0 00000400:00000200:0.0:1788574755.971148:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574755.971152:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574755.971156:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574755.971159:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574755.971161:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574755.971164:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574755.971166:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574755.971169:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547415:547415:256:4294967295] 192.168.202.128@tcp LPNI seq info [547338:547338:8:4294967295] 00000400:00000200:0.0:1788574755.971171:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574755.971176:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574755.971179:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574755.971182:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574755.971184:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574755.971186:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788574755.971193:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574755.971194:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574755.971197:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574755.971199:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574755.971201:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574755.971202:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574755.971204:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574755.971206:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516737664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.971212:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574755.971214:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516737664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574780 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.971226:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.971227:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574755.971230:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574755.971232:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574755.971235:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574755.971236:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310116009 00000100:00000001:0.0:1788574755.971238:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310119661 00000100:00000001:0.0:1788574755.971239:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310116009 00000100:00000001:0.0:1788574755.971240:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310119661 00000100:00000001:0.0:1788574755.971242:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574755.971243:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.971244:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574755.971247:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574755/real 1788574755] req@ffff8fc4ac821c00 x1875440516737664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574755.971252:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574755.971253:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574755.971254:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574755.971255:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574755.971256:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574755.971258:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574755.971260:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516737664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574755.971265:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516737664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.971269:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574755.971270:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.971271:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574755.971272:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574755.971274:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516737664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.971283:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574755.971286:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516737664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.971290:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574755.971291:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.971292:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574755.971293:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.971294:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574755.971296:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574755.971297:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574755.971300:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574755.971301:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574755.971303:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516737664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574755.971309:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516737664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574755.971312:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.971314:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574755.971317:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516737536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.971323:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574755.971325:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574755.971327:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574755.971330:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574755.971331:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574755.971333:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574755.971335:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574755.971337:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574755.971339:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574755.971345:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574755.971347:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788574755.971350:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574755.971351:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.971353:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574755.971357:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516737664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574780 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574755.971364:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574755.971366:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574755.971368:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574755.971370:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574755.971371:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574755.971373:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574755.971375:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574755.971376:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788574755.971380:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574755.971381:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574755.971382:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788574755.971384:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574755.971386:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.971387:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574755.971391:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574755.971392:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574755.971394:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574755.971395:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574756.480911:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574756.480920:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574756.480925:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574756.480932:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574756.480938:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574756.480943:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574756.480949:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574756.480953:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574756.480959:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574757.185002:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574757.185005:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574757.185008:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574757.505034:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574757.505040:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574757.505043:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574757.505048:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574757.505053:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574757.505058:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574757.505062:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574757.505065:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574757.505068:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574758.528916:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574758.528924:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574758.528930:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574758.528938:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574758.528943:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574758.528949:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.528962:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574758.528965:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574758.528968:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574759.233010:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574759.233013:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574759.233016:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574759.552957:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574759.552969:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574759.552974:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574759.552986:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574759.552992:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574759.552998:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574759.553009:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574759.553012:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574759.553015:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574759.808921:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310119661, at 4310119680 00000100:00000010:0.0:1788574759.808936:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574760.257033:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574760.257036:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574760.257038:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574760.577018:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574760.577026:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574760.577045:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574760.577050:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574760.577059:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574760.577066:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574760.577075:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574760.577080:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574760.577086:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574761.088978:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574761.088987:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574761.088989:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574761.088992:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574761.088996:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574761.089005:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15448 00000400:00000200:0.0:1788574761.089013:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574761.089019:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574761.089021:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574761.089024:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089026:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574761.089028:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089030:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574761.089033:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574761.089034:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574761.089036:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574761.089039:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574761.089041:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574761.089042:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574761.089044:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089046:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574761.089049:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089050:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574761.089053:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574761.089057:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089078:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574761.089081:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574761.089085:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574761.089088:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089091:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574761.089093:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574761.089100:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574761.089103:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574761.089107:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574761.089108:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089110:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574761.089112:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574761.089116:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574761.089124:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574761.089132:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574761.089142:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516737792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574761.089154:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574761.089167:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574761.089172:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516737792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574761.089193:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574761.089197:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574761.089201:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574761.089202:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574761.089204:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574761.089207:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574761.089210:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15448 00000400:00000200:0.0:1788574761.089215:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574761.089218:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574761.089221:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574761.089224:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089252:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574761.089254:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089257:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574761.089259:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574761.089261:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574761.089262:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574761.089264:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574761.089266:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574761.089269:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574761.089272:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089274:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574761.089277:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089279:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574761.089281:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574761.089283:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089286:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574761.089288:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574761.089294:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574761.089296:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089298:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574761.089309:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574761.089312:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089314:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574761.089317:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574761.089320:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574761.089323:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574761.089324:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089326:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574761.089328:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574761.089330:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574761.089334:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574761.089338:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574761.089349:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516737920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574761.089357:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574761.089359:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574761.089363:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516737920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574761.089372:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574761.089375:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574761.089379:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574761.089380:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574761.089382:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574761.089385:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574761.089388:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15448 00000400:00000200:0.0:1788574761.089392:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574761.089395:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574761.089397:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574761.089400:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089402:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574761.089405:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089407:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574761.089409:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574761.089411:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574761.089413:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574761.089415:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574761.089417:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574761.089419:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574761.089422:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089424:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574761.089426:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089431:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574761.089434:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574761.089436:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089439:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574761.089441:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574761.089444:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574761.089446:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089448:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574761.089450:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574761.089452:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089454:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574761.089457:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574761.089460:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574761.089462:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574761.089463:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089465:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574761.089467:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574761.089469:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574761.089472:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574761.089476:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574761.089480:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516738048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574761.089488:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574761.089490:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574761.089494:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516738048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574761.089502:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574761.089505:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574761.089509:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574761.089510:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574761.089512:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574761.089519:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574761.089522:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15448 00000400:00000200:0.0:1788574761.089526:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574761.089529:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574761.089532:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574761.089535:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089537:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574761.089540:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089542:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574761.089544:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574761.089546:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574761.089547:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574761.089568:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574761.089571:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574761.089574:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574761.089577:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089580:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574761.089582:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089584:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574761.089587:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574761.089590:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089593:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574761.089594:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574761.089598:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574761.089600:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089602:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574761.089604:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574761.089607:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089609:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574761.089613:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574761.089619:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574761.089622:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574761.089623:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089625:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574761.089627:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574761.089629:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574761.089633:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574761.089636:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574761.089641:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516738176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574761.089649:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574761.089652:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574761.089656:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516738176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574761.089664:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574761.089667:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574761.089671:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574761.089672:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574761.089674:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574761.089677:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574761.089679:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15448 00000400:00000200:0.0:1788574761.089683:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574761.089686:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574761.089688:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574761.089691:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089693:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574761.089695:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089697:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574761.089699:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574761.089701:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574761.089706:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574761.089708:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574761.089710:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574761.089713:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574761.089716:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089718:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574761.089721:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089723:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574761.089725:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574761.089728:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574761.089730:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574761.089732:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574761.089734:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574761.089736:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089738:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574761.089740:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574761.089743:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089745:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574761.089748:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574761.089750:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574761.089753:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574761.089754:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.089756:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574761.089758:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574761.089759:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574761.089763:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574761.089766:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574761.089771:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516738304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574761.089779:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574761.089781:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574761.089785:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516738304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574761.089799:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574761.089804:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15458) 00000100:00000001:0.0:1788574761.600912:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574761.600919:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574761.600922:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574761.600930:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574761.600937:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574761.600942:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574761.600948:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574761.600952:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574761.600956:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574761.600987:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574761.600990:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574761.600994:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574761.600997:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574761.601001:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574761.601004:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574761.601006:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574761.601007:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574761.601008:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574762.305009:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574762.305013:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574762.305017:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574762.624926:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574762.624931:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574762.624934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574762.624939:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574762.624946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574762.624950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574762.624954:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574762.624957:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574762.624962:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574763.648921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574763.648928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574763.648931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574763.648937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574763.648959:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574763.648964:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574763.648989:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574763.648992:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574763.648995:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574764.161110:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788574764.352983:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574764.352988:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574764.352992:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574764.672906:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574764.672916:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574764.672920:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574764.672927:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574764.672934:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574764.672946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574764.672958:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574764.672962:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574764.672968:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574764.928914:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310124781, at 4310124800 00000100:00000010:0.0:1788574764.928929:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf9c0. 00000020:00000001:0.0:1788574765.377013:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574765.377017:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574765.377020:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574765.696932:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574765.696937:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574765.696941:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574765.696948:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574765.696952:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574765.696956:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574765.696958:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574765.696960:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574765.696962:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574766.720942:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574766.720946:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574766.720949:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574766.720955:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574766.720960:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574766.720972:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574766.720983:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574766.720987:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574766.720993:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574767.425001:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574767.425005:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574767.425009:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574767.744967:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574767.744971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574767.744973:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574767.744976:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574767.744981:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574767.744985:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574767.744990:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574767.744994:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574767.745002:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574768.768943:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574768.768951:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574768.768955:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574768.768960:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574768.768966:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574768.768968:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574768.768990:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574768.768994:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574768.768997:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574769.473134:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574769.473139:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574769.473142:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574769.793060:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574769.793065:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574769.793070:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574769.793075:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574769.793081:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574769.793085:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574769.793405:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574769.793407:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574769.793410:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574770.048922:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310129901, at 4310129920 00000100:00000010:0.0:1788574770.048949:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfae0. 00000020:00000001:0.0:1788574770.497033:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574770.497036:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574770.497039:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574770.816917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574770.816927:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574770.816932:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574770.816948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574770.816955:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574770.816959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574770.816971:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574770.816973:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574770.816976:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574770.817001:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574770.817003:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574770.817005:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574770.817014:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574770.817019:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574770.817022:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574770.817025:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574770.817029:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574770.817032:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574771.840930:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574771.840936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574771.840942:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574771.840964:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574771.840969:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574771.840974:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574771.840979:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574771.840983:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574771.840986:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574771.841110:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574771.841117:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574771.841120:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574771.841123:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574771.841126:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574771.841130:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574771.841146:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574771.841148:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574771.841151:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574772.544998:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574772.545002:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574772.545010:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574772.864920:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574772.864930:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574772.864934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574772.864942:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574772.864949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574772.864953:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574772.864958:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574772.864964:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574772.865226:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574772.865260:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574772.865263:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574772.865264:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574772.865267:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574772.865270:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574772.865272:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574772.865274:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574772.865276:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574772.865278:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574773.888929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574773.888945:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574773.888950:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574773.888956:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574773.888962:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574773.888968:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574773.888985:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574773.888988:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574773.888992:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574774.593056:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574774.593061:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574774.593066:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574774.912924:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574774.912933:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574774.912952:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574774.912958:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574774.912962:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574774.912966:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574774.912969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574774.912972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574774.912975:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574774.913938:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574774.913941:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574774.913942:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574774.913945:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574774.913947:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574774.913949:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574774.913951:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574774.913953:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574774.913955:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574775.168930:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310135022, at 4310135040 00000100:00000010:0.0:1788574775.168944:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfc60. 00000020:00000001:0.0:1788574775.617022:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574775.617026:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574775.617029:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574775.936901:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574775.936906:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574775.936909:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574775.936914:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574775.936919:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574775.936924:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574775.936930:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574775.936933:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574775.936936:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574775.936961:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574775.936963:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574775.936965:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574775.936967:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574775.936973:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574775.936976:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574775.936984:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574775.936992:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574775.936994:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574776.960917:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574776.960921:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574776.960924:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574776.960929:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574776.960934:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574776.960937:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574776.960958:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574776.960960:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574776.960963:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574777.664961:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574777.664965:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574777.664968:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574777.984921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574777.984930:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574777.984933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574777.984942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574777.984950:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574777.984955:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574777.984960:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574777.984965:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574777.984971:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574777.984989:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574777.984994:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574777.984997:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574777.985003:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574777.985006:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574777.985009:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574777.985011:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574777.985013:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574777.985015:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574779.008912:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574779.008919:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574779.008923:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574779.008931:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574779.008937:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574779.008942:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574779.008960:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574779.008962:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574779.008964:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574779.008984:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574779.008986:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574779.008987:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574779.008989:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574779.008991:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574779.008993:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574779.008995:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574779.008996:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574779.008998:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574779.713032:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574779.713036:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574779.713039:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574780.032920:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574780.032929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574780.032933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574780.032940:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574780.032947:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574780.032953:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574780.032979:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574780.032982:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574780.032985:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574780.033038:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574780.033042:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574780.033043:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574780.033045:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574780.033048:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574780.033052:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574780.033056:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574780.033059:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574780.033063:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574780.288930:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310140141, at 4310140160 00000100:00000010:0.0:1788574780.288947:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf780. 00000020:00000001:0.0:1788574780.737089:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574780.737092:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574780.737101:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574781.056906:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574781.056911:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574781.056916:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574781.056923:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574781.056930:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574781.056951:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574781.056979:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574781.056983:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574781.056986:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574782.080924:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574782.080931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574782.080933:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574782.080941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574782.080945:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574782.080954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574782.080969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574782.080973:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574782.080975:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574782.785001:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574782.785004:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574782.785007:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574783.104923:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574783.104930:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574783.104934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574783.104941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574783.104948:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574783.104953:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574783.104959:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574783.104963:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574783.104967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574783.104998:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574783.105002:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574783.105004:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574783.105006:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574783.105009:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574783.105011:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574783.105013:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574783.105021:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574783.105023:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574784.128915:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574784.128922:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574784.128927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574784.128935:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574784.128939:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574784.128948:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574784.128954:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574784.128959:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574784.128962:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574784.833061:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574784.833064:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574784.833075:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574785.152987:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574785.152992:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574785.152994:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574785.152997:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574785.153000:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574785.153004:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574785.153008:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574785.153012:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574785.153019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574785.408915:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310145261, at 4310145280 00000100:00000010:0.0:1788574785.408933:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf2a0. 00000020:00000001:0.0:1788574785.857000:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574785.857004:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574785.857006:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574786.176928:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574786.176933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574786.176936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574786.176944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574786.176951:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574786.176955:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574786.176987:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574786.176990:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574786.176996:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574787.200910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574787.200917:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574787.200921:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574787.200928:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574787.200932:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574787.200937:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574787.200948:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574787.200952:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574787.200955:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574787.904961:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574787.904964:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574787.904966:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574788.224915:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574788.224923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574788.224927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574788.224934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574788.224939:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574788.224944:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574788.224948:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574788.224954:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574788.224959:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574789.248919:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574789.248927:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574789.248929:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574789.248936:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574789.248942:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574789.248946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574789.248950:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574789.248955:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574789.248958:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574789.952955:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574789.952958:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574789.952962:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574790.272903:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574790.272911:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574790.272913:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574790.272925:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574790.272930:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574790.272950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574790.272961:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574790.272968:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574790.272971:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574790.528890:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310150381, at 4310150400 00000100:00000010:0.0:1788574790.528903:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cff00. 00000020:00000001:0.0:1788574790.977023:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574790.977027:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574790.977030:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574791.296965:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574791.296971:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574791.296974:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574791.296988:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574791.296992:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574791.296996:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574791.296999:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574791.297001:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574791.297004:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574792.320958:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574792.320964:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574792.320968:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574792.320973:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574792.320977:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574792.320981:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574792.320986:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574792.320991:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574792.320998:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574793.025052:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574793.025059:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574793.025064:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574793.344953:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574793.344961:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574793.344967:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574793.344976:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574793.344985:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574793.344993:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574793.345002:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574793.345016:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574793.345019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574794.369062:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574794.369068:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574794.369069:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574794.369073:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574794.369077:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574794.369080:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574794.369083:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574794.369086:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574794.369088:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574795.073022:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574795.073026:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574795.073030:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574795.393011:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574795.393018:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574795.393022:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574795.393028:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574795.393034:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574795.393040:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574795.393083:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574795.393087:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574795.393090:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574795.648977:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310155501, at 4310155520 00000100:00000010:0.0:1788574795.649002:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf180. 00000020:00000001:0.0:1788574796.097020:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574796.097024:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574796.097026:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574796.416885:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574796.416891:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574796.416894:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574796.416899:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574796.416903:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574796.416906:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574796.416908:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574796.416911:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574796.416913:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.416952:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574796.416957:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574796.416960:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574796.416966:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574796.416972:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574796.416975:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574796.416978:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574796.416981:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574796.416984:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574796.928941:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574796.928950:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574796.928952:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574796.928955:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574796.928958:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574796.928964:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15484 00000400:00000200:0.0:1788574796.928971:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574796.928977:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574796.928980:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574796.928983:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.928987:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574796.928989:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.928992:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574796.928996:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574796.928998:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574796.929001:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574796.929004:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574796.929007:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574796.929009:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574796.929013:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929015:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574796.929017:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929018:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574796.929039:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574796.929043:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929046:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574796.929049:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574796.929052:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574796.929055:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.929057:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574796.929060:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574796.929066:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574796.929070:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574796.929073:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574796.929075:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.929079:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574796.929081:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574796.929086:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574796.929093:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574796.929101:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574796.929110:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516742272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574796.929120:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574796.929124:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574796.929129:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516742272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574796.929150:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574796.929153:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574796.929157:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574796.929160:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574796.929162:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574796.929165:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574796.929169:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15484 00000400:00000200:0.0:1788574796.929176:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574796.929193:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574796.929199:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574796.929205:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929208:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574796.929210:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929212:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574796.929215:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574796.929218:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574796.929222:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574796.929227:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574796.929230:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574796.929235:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574796.929243:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929249:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574796.929255:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929260:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574796.929264:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574796.929270:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929275:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574796.929281:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574796.929288:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574796.929292:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.929310:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574796.929315:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574796.929323:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.929328:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574796.929337:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574796.929344:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574796.929349:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574796.929352:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.929356:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574796.929366:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574796.929371:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574796.929378:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574796.929385:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574796.929397:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516742400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574796.929419:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574796.929424:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574796.929431:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516742400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574796.929445:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574796.929450:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574796.929475:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574796.929479:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574796.929482:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574796.929489:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574796.929495:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15484 00000400:00000200:0.0:1788574796.929502:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574796.929510:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574796.929515:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574796.929520:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929526:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574796.929531:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929537:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574796.929548:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574796.929551:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574796.929555:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574796.929560:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574796.929565:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574796.929571:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574796.929585:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929590:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574796.929594:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929596:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574796.929601:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574796.929608:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929613:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574796.929619:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574796.929625:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574796.929629:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.929633:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574796.929637:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574796.929642:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.929647:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574796.929654:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574796.929660:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574796.929665:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574796.929667:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.929670:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574796.929674:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574796.929677:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574796.929684:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574796.929692:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574796.929703:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516742528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574796.929717:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574796.929721:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574796.929733:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516742528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574796.929745:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574796.929748:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574796.929759:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574796.929761:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574796.929763:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574796.929768:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574796.929773:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15484 00000400:00000200:0.0:1788574796.929783:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574796.929789:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574796.929792:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574796.929799:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929803:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574796.929860:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929867:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574796.929874:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574796.929877:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574796.929881:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574796.929885:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574796.929891:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574796.929897:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574796.929905:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929911:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574796.929916:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929921:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574796.929927:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574796.929934:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574796.929940:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574796.929945:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574796.929951:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574796.929958:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.929961:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574796.929965:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574796.929992:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.929995:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574796.930000:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574796.930005:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574796.930011:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574796.930014:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.930017:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574796.930021:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574796.930024:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574796.930031:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574796.930039:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574796.930051:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516742656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574796.930066:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574796.930071:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574796.930079:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516742656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574796.930096:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574796.930101:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574796.930109:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574796.930113:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574796.930118:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574796.930126:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574796.930129:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15484 00000400:00000200:0.0:1788574796.930136:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574796.930144:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574796.930150:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574796.930158:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.930164:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574796.930169:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.930182:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574796.930187:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574796.930191:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574796.930195:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574796.930199:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574796.930202:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574796.930205:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574796.930211:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.930219:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574796.930224:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574796.930228:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574796.930234:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574796.930241:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574796.930244:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574796.930247:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574796.930253:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574796.930257:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.930261:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574796.930266:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574796.930272:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.930275:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574796.930283:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574796.930288:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574796.930294:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574796.930297:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574796.930299:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574796.930302:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574796.930306:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574796.930313:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574796.930319:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574796.930330:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516742784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574796.930362:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574796.930367:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574796.930375:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516742784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574796.930396:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574796.930406:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15494) 00000100:00000001:0.0:1788574797.440899:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574797.440908:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574797.440913:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574797.440922:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574797.440930:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574797.440945:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574797.440962:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574797.440965:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574797.440969:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574798.145034:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574798.145037:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574798.145040:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574798.464923:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574798.464931:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574798.464935:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574798.464943:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574798.464950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574798.464955:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574798.464971:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574798.464975:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574798.464979:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574799.488892:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574799.488899:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574799.488902:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574799.488908:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574799.488913:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574799.488920:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574799.488932:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574799.488954:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574799.488959:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574800.193033:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574800.193037:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574800.193042:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574800.768896:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310160621, at 4310160640 00000100:00000010:0.0:1788574800.768911:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf420. 00000020:00000001:0.0:1788574801.216983:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574801.216988:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574801.216992:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574801.536943:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574801.536950:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574801.536953:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574801.536958:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574801.536964:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574801.536968:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574801.536982:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574801.536985:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574801.536987:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574802.048953:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788574802.048971:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788574802.048979:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788574802.049159:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574802.049179:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574802.049185:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574802.049190:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574802.561034:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574802.561046:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574802.561054:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574802.561062:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574802.561066:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574802.561068:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574802.561102:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574802.561104:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574802.561111:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574803.264986:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574803.265016:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574803.265019:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574803.584931:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574803.584936:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574803.584940:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574803.584944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574803.584951:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574803.584954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574803.584959:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574803.584964:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574803.584970:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574804.608977:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574804.608987:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574804.608992:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574804.609001:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574804.609010:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574804.609019:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574804.609024:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574804.609036:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574804.609047:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574805.313099:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574805.313104:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574805.313108:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574805.633074:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574805.633081:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574805.633083:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574805.633092:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574805.633099:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574805.633105:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574805.633111:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574805.633115:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574805.633120:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574805.888964:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310165742, at 4310165760 00000100:00000010:0.0:1788574805.888987:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf360. 00000020:00000001:0.0:1788574806.337076:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574806.337080:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574806.337082:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574806.656906:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574806.656913:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574806.656916:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574806.656922:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574806.656926:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574806.656931:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574806.657088:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574806.657091:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574806.657093:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.680934:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574807.680943:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574807.680946:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574807.680953:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574807.680958:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574807.680964:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574807.680971:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574807.680975:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574807.680979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.681116:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574807.681118:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574807.681119:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574807.681122:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574807.681124:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574807.681140:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574807.681142:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574807.681143:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574807.681147:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574807.681170:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574807.681174:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574807.681176:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574807.681178:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574807.681181:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574807.681184:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574807.681186:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574807.681188:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574807.681191:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574808.385031:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574808.385035:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574808.385059:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574808.704897:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574808.704903:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574808.704908:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574808.704914:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574808.704920:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574808.704923:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574808.704941:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574808.704944:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574808.704946:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574809.728990:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574809.728998:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574809.729001:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574809.729007:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574809.729015:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574809.729022:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574809.729028:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574809.729031:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574809.729037:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574810.433074:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574810.433078:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574810.433082:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574811.008941:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310170861, at 4310170880 00000100:00000010:0.0:1788574811.008958:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf960. 00000020:00000001:0.0:1788574811.456973:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574811.456979:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574811.456982:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574811.777084:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574811.777087:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574811.777088:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574811.777091:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574811.777093:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574811.777094:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574811.777096:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574811.777097:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574811.777099:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574812.800913:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574812.800937:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574812.800951:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574812.800960:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574812.800965:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574812.800983:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574812.800987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574812.800990:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574812.800992:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574813.504978:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574813.504980:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574813.504982:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574813.824907:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574813.824916:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574813.824918:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574813.824924:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574813.824929:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574813.824934:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574813.824948:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574813.824953:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574813.824960:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574813.825002:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574813.825005:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574813.825007:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574813.825008:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574813.825011:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574813.825013:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574813.825015:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574813.825016:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574813.825018:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574814.848930:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574814.848937:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574814.848940:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574814.848950:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574814.848955:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574814.848962:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574814.848968:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574814.848973:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574814.848978:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574814.849014:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574814.849018:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574814.849020:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574814.849023:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574814.849026:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574814.849028:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574814.849032:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574814.849036:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574814.849041:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574815.553058:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574815.553060:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574815.553063:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574815.872917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574815.872933:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574815.872937:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574815.872943:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574815.872949:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574815.872954:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574815.872963:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574815.872967:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574815.872971:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574815.872998:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574815.873002:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574815.873003:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574815.873005:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574815.873007:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574815.873009:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574815.873011:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574815.873012:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574815.873014:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574816.128915:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310175982, at 4310176000 00000100:00000010:0.0:1788574816.128930:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfa20. 00000020:00000001:0.0:1788574816.577057:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574816.577060:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574816.577062:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574816.896960:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574816.896969:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574816.896989:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574816.896995:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574816.897066:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574816.897069:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574816.897073:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574816.897076:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574816.897081:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574817.920938:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574817.920942:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574817.920945:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574817.920947:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574817.920950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574817.920952:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574817.920954:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574817.920957:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574817.920959:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574818.624994:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574818.624997:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574818.624999:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574818.944938:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574818.944943:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574818.944947:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574818.944954:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574818.944960:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574818.944966:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574818.944981:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574818.944985:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574818.944988:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574819.969062:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574819.969067:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574819.969068:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574819.969071:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574819.969074:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574819.969077:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574819.969081:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574819.969082:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574819.969084:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574820.672950:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574820.672967:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574820.672970:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574820.992907:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574820.992913:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574820.992917:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574820.992927:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574820.992932:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574820.992936:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574820.992948:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574820.992950:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574820.992954:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574821.248905:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310181101, at 4310181120 00000100:00000010:0.0:1788574821.248918:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfcc0. 00000020:00000001:0.0:1788574821.697019:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574821.697021:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574821.697024:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574822.016916:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574822.016925:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574822.016928:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574822.016936:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574822.016943:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574822.016948:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574822.016962:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574822.016964:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574822.016969:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574823.040936:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574823.040941:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574823.040945:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574823.040949:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574823.040955:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574823.040960:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574823.040984:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574823.040988:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574823.040991:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574823.745039:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574823.745043:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574823.745047:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574824.064925:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574824.064929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574824.064934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574824.064942:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574824.064948:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574824.064952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574824.064972:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574824.064976:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574824.064979:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574825.089025:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574825.089028:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574825.089029:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574825.089031:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574825.089033:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574825.089035:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574825.090532:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574825.090535:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574825.090544:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574825.793013:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574825.793018:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574825.793021:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574826.112925:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574826.112931:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574826.112936:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574826.112945:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574826.112953:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574826.112959:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574826.112963:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574826.112968:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574826.112974:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574826.368912:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310186221, at 4310186240 00000100:00000010:0.0:1788574826.368928:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf900. 00000020:00000001:0.0:1788574826.817023:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574826.817028:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574826.817031:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574827.136957:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574827.136962:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574827.136982:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574827.136987:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574827.136994:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574827.137001:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574827.137007:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574827.137012:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574827.137015:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574828.160916:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574828.160921:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574828.160923:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574828.160929:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574828.160934:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574828.160938:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574828.160950:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574828.160953:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574828.160958:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574828.161037:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574828.161040:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574828.161041:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574828.161043:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574828.161045:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574828.161048:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574828.161050:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574828.161051:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574828.161053:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574828.864986:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574828.864990:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574828.864993:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574829.184920:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574829.184928:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574829.184931:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574829.184937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574829.184943:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574829.184948:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574829.184953:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574829.184957:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574829.184963:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574830.209032:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574830.209058:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574830.209060:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574830.209065:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574830.209071:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574830.209076:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574830.209081:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574830.209086:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574830.209091:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574830.913070:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574830.913076:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574830.913082:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574831.232938:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574831.232943:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574831.232947:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574831.232956:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574831.232963:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574831.232973:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574831.232990:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574831.232992:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574831.232995:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574831.488910:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310191341, at 4310191360 00000100:00000010:0.0:1788574831.488933:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfc00. 00000020:00000001:0.0:1788574831.937067:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574831.937071:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574831.937073:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574832.257194:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574832.257199:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574832.257201:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574832.257204:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574832.257207:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574832.257209:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574832.257211:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574832.257213:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574832.257215:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574832.768943:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574832.768955:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574832.768981:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574832.768985:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574832.768989:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574832.768997:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15520 00000400:00000200:0.0:1788574832.769008:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574832.769014:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574832.769017:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574832.769020:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769023:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574832.769025:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769027:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574832.769031:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574832.769032:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574832.769035:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574832.769038:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574832.769039:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574832.769041:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574832.769043:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769044:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574832.769046:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769047:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574832.769049:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574832.769052:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769053:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574832.769055:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574832.769059:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574832.769061:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.769063:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574832.769065:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574832.769072:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574832.769075:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574832.769082:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574832.769083:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.769087:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574832.769090:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574832.769095:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574832.769103:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574832.769110:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574832.769128:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516746752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574832.769137:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574832.769141:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574832.769146:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516746752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574832.769176:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574832.769180:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574832.769183:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574832.769185:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574832.769187:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574832.769191:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574832.769194:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15520 00000400:00000200:0.0:1788574832.769198:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574832.769203:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574832.769205:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574832.769209:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769217:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574832.769228:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769236:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574832.769241:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574832.769246:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574832.769255:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574832.769261:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574832.769274:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574832.769283:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574832.769300:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769305:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574832.769312:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769316:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574832.769319:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574832.769323:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769324:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574832.769326:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574832.769333:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574832.769336:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.769344:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574832.769349:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574832.769359:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.769361:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574832.769369:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574832.769375:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574832.769378:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574832.769379:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.769382:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574832.769389:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574832.769393:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574832.769398:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574832.769403:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574832.769415:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516746880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574832.769488:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574832.769490:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574832.769494:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516746880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574832.769534:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574832.769538:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574832.769546:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574832.769549:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574832.769551:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574832.769553:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574832.769559:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15520 00000400:00000200:0.0:1788574832.769569:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574832.769574:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574832.769577:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574832.769579:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769581:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574832.769582:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769583:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574832.769585:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574832.769586:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574832.769591:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574832.769596:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574832.769598:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574832.769601:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574832.769622:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769633:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574832.769639:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769647:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574832.769658:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574832.769667:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769673:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574832.769685:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574832.769693:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574832.769706:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.769711:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574832.769716:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574832.769728:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.769731:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574832.769737:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574832.769742:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574832.769744:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574832.769745:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.769746:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574832.769752:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574832.769758:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574832.769765:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574832.769769:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574832.769773:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516747008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574832.769780:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574832.769785:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574832.769791:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516747008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574832.769852:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574832.769856:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574832.769860:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574832.769865:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574832.769875:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574832.769881:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574832.769888:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15520 00000400:00000200:0.0:1788574832.769900:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574832.769907:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574832.769915:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574832.769932:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769936:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574832.769947:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.769958:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574832.769965:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574832.769970:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574832.769979:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574832.769986:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574832.769995:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574832.770003:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574832.770016:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.770020:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574832.770028:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.770036:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574832.770043:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574832.770052:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574832.770055:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574832.770056:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574832.770063:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574832.770066:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.770074:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574832.770077:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574832.770079:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.770080:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574832.770084:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574832.770089:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574832.770095:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574832.770099:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.770101:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574832.770104:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574832.770108:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574832.770118:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574832.770133:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574832.770143:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516747136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574832.770161:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574832.770166:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574832.770175:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516747136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574832.770193:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574832.770197:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574832.770200:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574832.770206:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574832.770210:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574832.770216:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574832.770222:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15520 00000400:00000200:0.0:1788574832.770229:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574832.770236:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574832.770241:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574832.770249:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.770257:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574832.770262:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.770278:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574832.770284:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574832.770287:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574832.770292:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574832.770297:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574832.770304:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574832.770311:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574832.770322:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.770327:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574832.770348:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574832.770353:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574832.770362:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574832.770368:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574832.770374:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574832.770381:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574832.770387:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574832.770394:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.770399:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574832.770405:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574832.770411:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.770418:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574832.770424:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574832.770430:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574832.770445:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574832.770449:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574832.770456:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574832.770462:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574832.770468:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574832.770492:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574832.770501:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574832.770516:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516747264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574832.770537:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574832.770546:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574832.770562:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516747264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574832.770583:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574832.770591:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15530) 00000100:00000001:0.0:1788574833.280921:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574833.280929:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574833.280934:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574833.280962:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574833.280969:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574833.280974:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574833.280994:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574833.280998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574833.281002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574833.985039:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574833.985044:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574833.985047:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574834.304913:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574834.304920:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574834.304924:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574834.304934:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574834.304941:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574834.304947:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574834.304951:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574834.304955:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574834.304961:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574835.328959:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574835.328964:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574835.328967:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574835.328975:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574835.328981:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574835.328992:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.329004:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574835.329007:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.329011:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574835.842238:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574835.842242:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574835.842245:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574835.842253:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574832/real 1788574835] req@ffff8fc4b80c0700 x1875440516747136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574835.842266:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574835.842268:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574835.842272:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c45478. 00000400:00000200:0.0:1788574835.842275:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcc38 00000400:00000010:0.0:1788574835.842283:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcc38. 00000100:00000001:0.0:1788574835.842287:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574835.842290:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c0700 x1875440516747136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574835.842299:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c0700 x1875440516747136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842306:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574835.842309:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842311:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574835.842314:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574835.842318:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c0700 x1875440516747136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842325:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574835.842330:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c0700 x1875440516747136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842337:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574835.842341:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842342:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574835.842344:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842347:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574835.842350:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574835.842354:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574835.842359:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574835.842362:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574835.842366:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c0700 x1875440516747136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574835.842374:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516747136:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574835.842380:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574835.842388:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574832/real 1788574835] req@ffff8fc4b80c2300 x1875440516747264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574835.842412:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574835.842414:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574835.842416:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c45d68. 00000400:00000200:0.0:1788574835.842419:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fccc0 00000400:00000010:0.0:1788574835.842420:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fccc0. 00000100:00000001:0.0:1788574835.842423:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574835.842428:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c2300 x1875440516747264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574835.842436:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c2300 x1875440516747264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842442:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574835.842443:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842445:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574835.842447:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574835.842450:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c2300 x1875440516747264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842458:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574835.842462:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c2300 x1875440516747264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842469:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574835.842471:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842472:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574835.842474:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842476:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574835.842479:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574835.842481:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574835.842485:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574835.842487:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574835.842498:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c2300 x1875440516747264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574835.842506:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516747264:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574835.842510:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574835.842513:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574832/real 1788574835] req@ffff8fc4b80c2680 x1875440516746880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574835.842523:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574835.842525:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574835.842527:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c458f0. 00000400:00000200:0.0:1788574835.842529:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc088 00000400:00000010:0.0:1788574835.842531:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc088. 00000100:00000001:0.0:1788574835.842534:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574835.842537:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c2680 x1875440516746880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574835.842555:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c2680 x1875440516746880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842562:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574835.842564:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842566:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574835.842567:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574835.842570:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c2680 x1875440516746880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842576:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574835.842580:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c2680 x1875440516746880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842587:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574835.842589:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842595:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574835.842597:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842598:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574835.842600:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574835.842603:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574835.842607:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574835.842610:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574835.842613:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c2680 x1875440516746880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574835.842621:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516746880:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574835.842625:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574835.842628:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574832/real 1788574835] req@ffff8fc4b80c1f80 x1875440516747008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574835.842636:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574835.842638:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574835.842639:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c45790. 00000400:00000200:0.0:1788574835.842642:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc440 00000400:00000010:0.0:1788574835.842644:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc440. 00000100:00000001:0.0:1788574835.842647:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574835.842650:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c1f80 x1875440516747008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574835.842658:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c1f80 x1875440516747008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842665:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574835.842666:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842668:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574835.842670:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574835.842674:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c1f80 x1875440516747008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842692:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574835.842697:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c1f80 x1875440516747008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842704:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574835.842706:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842708:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574835.842710:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842711:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574835.842714:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574835.842717:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574835.842721:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574835.842724:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574835.842728:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c1f80 x1875440516747008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574835.842736:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516747008:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574835.842741:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574835.842745:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574832/real 1788574835] req@ffff8fc4b80c1c00 x1875440516746752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574835.842753:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574835.842754:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574835.842757:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c45370. 00000400:00000200:0.0:1788574835.842760:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fce58 00000400:00000010:0.0:1788574835.842762:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fce58. 00000100:00000001:0.0:1788574835.842765:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574835.842768:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4b80c1c00 x1875440516746752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574835.842791:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4b80c1c00 x1875440516746752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.842798:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574835.842800:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.842802:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574835.842803:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574835.843441:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4b80c1c00 x1875440516746752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.843451:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574835.843457:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4b80c1c00 x1875440516746752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.843464:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574835.843467:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.843468:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574835.843469:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.843471:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574835.843472:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574835.843474:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574835.843477:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574835.843479:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574835.843481:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4b80c1c00 x1875440516746752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574835.843486:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4b80c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516746752:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574835.843490:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.843492:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574835.843494:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c0700 x1875440516747136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.843510:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574835.843512:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574835.843515:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788574835.843519:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574835.843521:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574835.843523:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574835.843530:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574835.843532:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788574835.843534:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574835.843536:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574835.843537:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c0700. 00000100:00000001:0.0:1788574835.843541:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574835.843542:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.843546:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574835.843550:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c2300 x1875440516747264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.843558:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574835.843559:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574835.843561:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788574835.843564:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574835.843566:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574835.843567:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574835.843569:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574835.843571:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574835.843574:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574835.843575:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574835.843576:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c2300. 00000100:00000001:0.0:1788574835.843579:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574835.843580:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.843582:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574835.843586:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c2680 x1875440516746880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.843593:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574835.843594:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574835.843602:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788574835.843604:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574835.843606:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574835.843607:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574835.843609:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574835.843611:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574835.843614:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574835.843619:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574835.843621:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c2680. 00000100:00000001:0.0:1788574835.843624:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574835.843625:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.843627:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574835.843632:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c1f80 x1875440516747008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.843638:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574835.843640:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574835.843642:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788574835.843644:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574835.843645:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574835.843646:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574835.843647:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574835.843648:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574835.843650:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574835.843651:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574835.843651:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c1f80. 00000100:00000001:0.0:1788574835.843653:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574835.843654:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.843655:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574835.843657:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4b80c1c00 x1875440516746752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574835.843661:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574835.843661:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574835.843662:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788574835.843664:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574835.843665:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574835.843666:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574835.843674:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574835.843675:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574835.843678:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574835.843679:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574835.843680:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4b80c1c00. 00000100:00000001:0.0:1788574835.843683:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574835.843684:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.843687:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574835.843696:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574835.843698:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574835.843701:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574835.843703:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574836.033004:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574836.033009:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574836.033012:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574836.353005:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574836.353012:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574836.353018:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574836.353023:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574836.353027:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574836.353029:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574836.353031:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574836.353032:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574836.353034:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574836.608949:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310196461, at 4310196480 00000100:00000010:0.0:1788574836.608968:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf0c0. 00000020:00000001:0.0:1788574837.057072:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574837.057076:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574837.057079:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574837.377000:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574837.377006:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574837.377013:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574837.377019:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574837.377032:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574837.377040:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.377067:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574837.377072:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.377100:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.889427:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574837.889444:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574837.889452:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574837.889494:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516747392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.889549:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574837.889556:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574837.889564:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574837.889579:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574837.889585:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574837.889597:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516747392:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574837.889609:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574837.889621:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574837.889628:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574837.889635:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574837.889652:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574837.889658:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574837.889674:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574837.889683:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574837.889698:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574837.889713:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516747392, portal 25 00000100:00000001:0.0:1788574837.889720:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574837.889727:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788574837.889752:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516747392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.889784:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574837.889795:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574837.889801:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574837.889852:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516747392, offset 0 00000400:00000200:0.0:1788574837.889862:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574837.889872:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574837.889885:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574837.889920:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574837.889930:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574837.889944:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574837.889953:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574837.889969:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547491:547491:256:4294967295] 192.168.202.128@tcp LPNI seq info [547414:547414:8:4294967295] 00000400:00000200:0.0:1788574837.889992:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574837.890035:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574837.890065:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574837.890087:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574837.890120:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574837.890130:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574837.890165:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574837.890171:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574837.890188:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574837.890206:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574837.890219:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574837.890227:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574837.890242:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574837.890255:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516747392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.890294:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574837.890304:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516747392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.890337:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.890357:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574837.890368:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574837.890380:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574837.890387:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310201582 00000100:00000010:0.0:1788574837.890400:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574837.890412:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574837.890435:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310201582 00000100:00000010:0.0:1788574837.890442:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50f00. 00000100:00000001:0.0:1788574837.890449:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574837.890458:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574837.890469:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.890478:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574837.890496:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574837/real 1788574837] req@ffff8fc4ac820000 x1875440516747392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574837.890535:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574837.890544:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574837.890551:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574837.890564:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574837.890573:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574837.890585:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574837.890596:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516747392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574837.890640:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516747392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.890666:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574837.890669:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.890682:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574837.890690:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574837.890702:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516747392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.890737:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574837.890748:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516747392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.890773:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574837.890782:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.890787:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574837.890804:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.890849:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574837.890857:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574837.890873:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574837.890890:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574837.890899:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574837.890915:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516747392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574837.890945:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516747392:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574837.890954:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.890959:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574837.890963:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516747392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.890972:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574837.890975:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574837.890979:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574837.890984:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574837.890988:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574837.890991:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574837.890994:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574837.890997:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788574837.891001:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574837.891003:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574837.891006:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788574837.891011:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574837.891012:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.891015:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.891024:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574837.891026:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.891029:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574837.891031:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574837.891033:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574837.891038:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516747520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891052:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574837.891054:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574837.891057:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574837.891060:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574837.891062:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574837.891066:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516747520:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574837.891071:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574837.891074:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574837.891076:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574837.891078:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574837.891081:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574837.891083:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574837.891088:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574837.891092:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574837.891097:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574837.891101:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516747520, portal 10 00000100:00000001:0.0:1788574837.891104:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574837.891106:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788574837.891113:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516747520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891121:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574837.891124:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574837.891126:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574837.891129:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516747520, offset 0 00000400:00000200:0.0:1788574837.891134:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574837.891139:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574837.891144:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574837.891148:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574837.891151:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574837.891155:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574837.891158:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574837.891165:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547492:547492:256:4294967295] 192.168.202.128@tcp LPNI seq info [547415:547415:8:4294967295] 00000400:00000200:0.0:1788574837.891171:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574837.891179:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574837.891185:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574837.891188:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574837.891200:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574837.891203:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574837.891217:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574837.891219:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574837.891223:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574837.891227:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574837.891230:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574837.891232:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574837.891235:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574837.891239:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516747520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891248:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574837.891252:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516747520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891259:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.891261:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574837.891264:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574837.891268:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574837.891271:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574837.891274:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310197780 00000100:00000001:0.0:1788574837.891277:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310201582 00000100:00000001:0.0:1788574837.891279:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310197780 00000100:00000001:0.0:1788574837.891282:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310201582 00000100:00000001:0.0:1788574837.891284:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574837.891290:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.891292:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574837.891296:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574837/real 1788574837] req@ffff8fc4ac823100 x1875440516747520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574837.891306:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574837.891307:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574837.891309:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574837.891311:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574837.891313:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574837.891315:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574837.891319:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516747520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574837.891328:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516747520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891335:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574837.891337:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.891339:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574837.891340:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574837.891360:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516747520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891367:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574837.891372:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516747520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891379:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574837.891381:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.891383:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574837.891384:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.891386:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574837.891389:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574837.891392:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574837.891400:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574837.891403:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574837.891407:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516747520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574837.891416:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516747520:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574837.891421:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574837.891425:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516747648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891433:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574837.891434:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574837.891437:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574837.891439:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574837.891441:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574837.891444:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516747648:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574837.891447:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574837.891450:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574837.891452:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574837.891453:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574837.891456:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574837.891459:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574837.891462:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574837.891465:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574837.891469:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574837.891472:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516747648, portal 10 00000100:00000001:0.0:1788574837.891474:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574837.891475:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788574837.891482:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516747648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891489:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574837.891497:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574837.891500:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574837.891503:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516747648, offset 0 00000400:00000200:0.0:1788574837.891507:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574837.891512:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574837.891515:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574837.891518:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574837.891521:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574837.891523:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574837.891526:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574837.891530:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547493:547493:256:4294967295] 192.168.202.128@tcp LPNI seq info [547416:547416:8:4294967295] 00000400:00000200:0.0:1788574837.891534:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574837.891542:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574837.891547:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574837.891550:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574837.891554:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574837.891557:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574837.891569:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574837.891571:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574837.891574:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574837.891578:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574837.891581:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574837.891583:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574837.891586:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574837.891590:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516747648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891599:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574837.891602:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516747648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891610:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.891615:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574837.891619:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574837.891622:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574837.891626:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574837.891628:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310197787 00000100:00000001:0.0:1788574837.891631:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310201582 00000100:00000001:0.0:1788574837.891634:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310197787 00000100:00000001:0.0:1788574837.891636:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310201582 00000100:00000001:0.0:1788574837.891638:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574837.891641:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.891643:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574837.891646:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574837/real 1788574837] req@ffff8fc4ac822a00 x1875440516747648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574837.891655:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574837.891656:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574837.891658:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574837.891660:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574837.891662:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574837.891665:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574837.891668:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516747648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574837.891677:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516747648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891684:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574837.891686:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.891688:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574837.891689:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574837.891693:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516747648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891700:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574837.891710:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516747648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891718:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574837.891721:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.891723:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574837.891724:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.891726:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574837.891730:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574837.891732:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574837.891736:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574837.891738:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574837.891742:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516747648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574837.891750:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516747648:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574837.891754:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574837.891758:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516747776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891772:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574837.891774:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574837.891777:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574837.891778:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574837.891781:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574837.891784:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516747776:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574837.891788:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574837.891790:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574837.891792:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574837.891794:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574837.891797:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574837.891799:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574837.891834:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574837.891838:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574837.891842:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574837.891845:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516747776, portal 4 00000100:00000001:0.0:1788574837.891847:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574837.891848:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788574837.891855:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516747776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891862:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574837.891865:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574837.891867:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574837.891870:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516747776, offset 0 00000400:00000200:0.0:1788574837.891873:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574837.891877:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574837.891881:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574837.891884:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574837.891887:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574837.891890:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574837.891892:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574837.891896:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547494:547494:256:4294967295] 192.168.202.128@tcp LPNI seq info [547417:547417:8:4294967295] 00000400:00000200:0.0:1788574837.891900:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574837.891907:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574837.891912:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574837.891915:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574837.891919:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574837.891922:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574837.891933:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574837.891935:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574837.891938:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574837.891942:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574837.891946:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574837.891951:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574837.891954:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574837.891958:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516747776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891967:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574837.891970:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516747776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.891978:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.891980:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574837.891983:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574837.891986:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574837.891989:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574837.891991:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310197928 00000100:00000001:0.0:1788574837.891994:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310201582 00000100:00000001:0.0:1788574837.891997:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310197928 00000100:00000001:0.0:1788574837.891999:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310201582 00000100:00000001:0.0:1788574837.892001:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574837.892003:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.892005:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574837.892008:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574837/real 1788574837] req@ffff8fc4ac823b80 x1875440516747776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574837.892017:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574837.892018:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574837.892020:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574837.892022:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574837.892024:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574837.892026:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574837.892030:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516747776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574837.892039:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516747776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892050:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574837.892052:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.892054:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574837.892055:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574837.892059:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516747776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892067:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574837.892071:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516747776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892078:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574837.892081:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.892083:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574837.892084:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.892086:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574837.892088:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574837.892090:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574837.892094:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574837.892096:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574837.892100:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516747776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574837.892108:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516747776:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574837.892112:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574837.892116:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516747904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892124:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574837.892125:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574837.892131:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574837.892133:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574837.892135:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574837.892139:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516747904:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574837.892142:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574837.892144:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574837.892146:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574837.892148:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574837.892151:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574837.892153:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574837.892156:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574837.892159:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574837.892162:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574837.892164:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516747904, portal 4 00000100:00000001:0.0:1788574837.892167:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574837.892168:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788574837.892174:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516747904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892181:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574837.892183:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574837.892185:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574837.892187:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516747904, offset 0 00000400:00000200:0.0:1788574837.892191:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574837.892195:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574837.892198:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574837.892201:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574837.892204:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574837.892206:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574837.892209:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574837.892213:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547495:547495:256:4294967295] 192.168.202.128@tcp LPNI seq info [547418:547418:8:4294967295] 00000400:00000200:0.0:1788574837.892217:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574837.892224:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574837.892232:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574837.892235:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574837.892239:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574837.892242:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788574837.892253:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574837.892255:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574837.892258:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574837.892262:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574837.892265:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574837.892267:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574837.892269:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574837.892273:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516747904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892281:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574837.892284:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516747904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892291:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.892293:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574837.892295:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574837.892298:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574837.892301:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574837.892303:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310197930 00000100:00000001:0.0:1788574837.892306:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310201582 00000100:00000001:0.0:1788574837.892308:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310197930 00000100:00000001:0.0:1788574837.892310:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310201582 00000100:00000001:0.0:1788574837.892313:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574837.892315:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.892317:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574837.892320:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574837/real 1788574837] req@ffff8fc4ac821500 x1875440516747904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574837.892333:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574837.892334:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574837.892336:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574837.892338:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574837.892340:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574837.892355:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574837.892359:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516747904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574837.892369:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516747904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892376:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574837.892377:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.892379:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574837.892381:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574837.892384:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516747904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892392:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574837.892396:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516747904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892404:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574837.892406:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.892408:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574837.892409:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.892411:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574837.892413:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574837.892415:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574837.892419:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574837.892421:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574837.892431:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516747904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574837.892439:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516747904:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574837.892443:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.892445:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574837.892448:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516747520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892456:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574837.892458:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574837.892460:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574837.892463:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574837.892465:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574837.892466:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574837.892468:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574837.892470:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788574837.892472:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574837.892474:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574837.892476:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788574837.892479:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574837.892480:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.892482:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574837.892485:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516747648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892492:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574837.892493:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574837.892495:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574837.892497:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574837.892498:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574837.892500:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574837.892502:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574837.892503:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788574837.892505:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574837.892511:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574837.892513:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788574837.892516:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574837.892517:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.892519:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574837.892523:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516747776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892529:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574837.892530:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574837.892532:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574837.892535:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574837.892536:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574837.892537:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574837.892539:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574837.892541:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788574837.892543:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574837.892544:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574837.892546:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788574837.892548:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574837.892549:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.892551:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574837.892554:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516747904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574837.892561:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574837.892562:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574837.892564:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574837.892566:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574837.892567:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574837.892569:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574837.892571:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574837.892572:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788574837.892575:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574837.892576:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574837.892577:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788574837.892580:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574837.892581:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.892586:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574837.892591:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574837.892593:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574837.892595:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574837.892597:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574838.400929:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574838.400937:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574838.400941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574838.400949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574838.400955:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574838.400959:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574838.400976:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574838.400980:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574838.400985:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574838.401024:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574838.401031:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574838.401035:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574838.401045:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574838.401052:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574838.401059:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574838.401065:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574838.401069:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574838.401082:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574839.105066:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574839.105070:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574839.105074:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574839.425025:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574839.425035:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574839.425042:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574839.425056:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574839.425066:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574839.425072:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574839.425085:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574839.425089:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574839.425093:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574839.425143:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574839.425147:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574839.425149:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574839.425166:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574839.425168:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574839.425170:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574839.425171:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574839.425173:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574839.425174:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574840.448935:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574840.448941:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574840.448946:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574840.448954:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574840.448960:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574840.448964:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574840.448978:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574840.448982:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574840.448985:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574840.960944:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788574840.960952:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788574840.960957:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788574840.961108:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574840.961123:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574840.961127:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574840.961130:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788574841.153040:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574841.153044:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574841.153046:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574841.472971:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574841.472981:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574841.472986:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574841.472993:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574841.472998:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574841.473003:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574841.473036:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574841.473039:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574841.473044:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574841.728914:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310201582, at 4310201600 00000100:00000010:0.0:1788574841.728936:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574842.177054:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574842.177062:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574842.177066:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574842.496974:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574842.496983:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574842.496990:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574842.497003:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574842.497018:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574842.497031:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574842.497044:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574842.497050:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574842.497060:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574843.008980:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574843.008995:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574843.008999:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574843.009004:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574843.009009:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574843.009020:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15530 00000400:00000200:0.0:1788574843.009032:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574843.009039:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574843.009044:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574843.009047:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009050:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574843.009053:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009055:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574843.009060:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574843.009062:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574843.009065:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574843.009069:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574843.009071:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574843.009074:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574843.009077:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009079:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574843.009104:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009106:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574843.009109:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574843.009113:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009115:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574843.009119:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574843.009122:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574843.009126:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.009129:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574843.009131:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574843.009139:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574843.009143:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574843.009148:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574843.009150:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.009153:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574843.009157:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574843.009163:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574843.009173:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574843.009182:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574843.009193:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516748032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574843.009205:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574843.009211:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574843.009230:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516748032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574843.009255:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574843.009259:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574843.009262:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574843.009264:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574843.009266:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574843.009270:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574843.009279:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15530 00000400:00000200:0.0:1788574843.009285:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574843.009290:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574843.009297:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574843.009303:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009312:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574843.009319:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009324:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574843.009328:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574843.009330:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574843.009334:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574843.009340:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574843.009351:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574843.009359:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574843.009384:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009392:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574843.009397:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009402:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574843.009413:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574843.009425:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009433:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574843.009441:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574843.009451:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574843.009460:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.009465:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574843.009476:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574843.009485:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.009496:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574843.009509:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574843.009516:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574843.009525:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574843.009540:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.009548:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574843.009556:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574843.009561:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574843.009569:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574843.009580:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574843.009593:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516748160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574843.009620:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574843.009629:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574843.009643:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516748160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574843.009688:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574843.009694:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574843.009707:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574843.009712:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574843.009715:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574843.009718:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574843.009727:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15530 00000400:00000200:0.0:1788574843.009744:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574843.009753:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574843.009758:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574843.009766:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009776:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574843.009783:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009787:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574843.009794:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574843.009801:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574843.009805:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574843.009848:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574843.009854:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574843.009867:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574843.009885:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009893:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574843.009899:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009905:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574843.009911:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574843.009921:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574843.009927:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574843.009932:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574843.009939:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574843.009947:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.009951:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574843.009955:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574843.009958:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.009959:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574843.009971:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574843.009979:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574843.009985:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574843.009989:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.010003:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574843.010007:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574843.010014:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574843.010025:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574843.010034:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574843.010048:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516748288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574843.010068:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574843.010075:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574843.010089:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516748288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574843.010135:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574843.010151:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574843.010156:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574843.010157:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574843.010159:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574843.010169:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574843.010176:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15530 00000400:00000200:0.0:1788574843.010185:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574843.010198:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574843.010204:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574843.010213:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.010235:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574843.010243:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.010249:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574843.010255:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574843.010257:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574843.010260:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574843.010266:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574843.010275:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574843.010288:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574843.010300:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.010314:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574843.010323:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.010328:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574843.010335:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574843.010341:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574843.010348:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574843.010353:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574843.010361:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574843.010367:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.010374:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574843.010387:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574843.010396:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.010402:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574843.010411:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574843.010419:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574843.010432:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574843.010438:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.010446:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574843.010450:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574843.010456:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574843.010464:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574843.010470:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574843.010487:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516748416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574843.010512:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574843.010520:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574843.010532:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516748416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574843.010557:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574843.010563:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574843.010572:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574843.010579:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574843.010584:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574843.010594:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574843.010605:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15530 00000400:00000200:0.0:1788574843.010616:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574843.010624:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574843.010632:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574843.010646:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.010654:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574843.010660:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.010675:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574843.010682:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574843.010687:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574843.010695:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574843.010701:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574843.010712:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574843.010719:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574843.010730:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.010740:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574843.010747:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574843.010756:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574843.010768:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574843.010781:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574843.010790:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574843.010804:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574843.010841:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574843.010845:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.010847:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574843.010849:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574843.010856:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.010858:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574843.010867:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574843.010873:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574843.010879:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574843.010882:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574843.010889:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574843.010894:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574843.010898:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574843.010908:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574843.010916:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574843.010930:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516748544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574843.010978:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574843.010984:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574843.010995:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516748544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574843.011018:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574843.011027:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15540) 00000020:00000001:0.0:1788574844.224986:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574844.224989:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574844.224992:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574844.544982:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574844.544986:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574844.544987:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574844.544990:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574844.544994:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574844.545000:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574844.545020:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574844.545024:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574844.545028:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574845.569061:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574845.569067:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574845.569073:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574845.569079:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574845.569086:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574845.569091:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574845.569096:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574845.569098:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574845.569101:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574846.273041:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574846.273045:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574846.273047:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574846.592986:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574846.592990:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574846.592994:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574846.592998:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574846.593014:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574846.593019:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574846.593023:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574846.593027:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574846.593032:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574846.849118:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310206702, at 4310206720 00000100:00000010:0.0:1788574846.849149:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf480. 00000020:00000001:0.0:1788574847.296991:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574847.296994:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574847.296997:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574847.616882:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574847.616890:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574847.616893:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574847.616899:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574847.616904:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574847.616910:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574847.616917:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574847.616921:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574847.616927:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574847.616957:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574847.616961:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574847.616965:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574847.616968:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574847.616972:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574847.616976:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574847.616978:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574847.616981:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574847.616984:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574848.640922:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574848.640928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574848.640932:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574848.640937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574848.640944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574848.640951:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574848.640972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574848.640976:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574848.640981:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574849.345042:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574849.345047:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574849.345050:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574849.664922:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574849.664929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574849.664933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574849.664942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574849.664950:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574849.664956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574849.664974:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574849.664977:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574849.664979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574849.665007:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574849.665011:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574849.665012:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574849.665014:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574849.665017:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574849.665019:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574849.665021:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574849.665023:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574849.665024:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574850.688910:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574850.688918:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574850.688922:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574850.688931:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574850.688938:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574850.688944:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574850.688959:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574850.688962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574850.688965:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574850.688994:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574850.688999:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574850.689003:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574850.689006:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574850.689010:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574850.689013:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574850.689015:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574850.689017:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574850.689050:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574851.393082:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574851.393085:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574851.393089:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574851.712916:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574851.712925:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574851.712932:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574851.712939:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574851.712947:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574851.712953:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574851.712966:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574851.712969:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574851.712972:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574851.968869:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310211821, at 4310211840 00000100:00000010:0.0:1788574851.968882:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf000. 00000020:00000001:0.0:1788574852.417019:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574852.417023:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574852.417026:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574852.736905:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574852.736912:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574852.736915:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574852.736920:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574852.736924:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574852.736929:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574852.736933:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574852.736938:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574852.736945:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574852.737018:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574852.737023:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574852.737025:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574852.737028:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574852.737031:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574852.737033:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574852.737036:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574852.737038:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574852.737041:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574853.760909:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574853.760929:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574853.760936:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574853.760945:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574853.760966:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574853.760970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574853.760973:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574853.760978:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574853.760981:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574853.761014:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574853.761019:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574853.761020:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574853.761024:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574853.761026:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574853.761029:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574853.761031:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574853.761034:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574853.761037:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574854.465024:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574854.465029:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574854.465032:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574854.784908:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574854.784917:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574854.784922:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574854.784951:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574854.784958:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574854.784964:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574854.784969:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574854.784973:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574854.784979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574854.784997:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574854.785000:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574854.785003:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574854.785010:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574854.785022:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574854.785025:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574854.785027:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574854.785029:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574854.785032:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574855.808927:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574855.808936:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574855.808939:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574855.808947:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574855.808956:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574855.808962:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574855.808978:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574855.808983:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574855.808988:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574855.809008:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574855.809013:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574855.809015:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574855.809019:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574855.809022:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574855.809025:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574855.809027:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574855.809030:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574855.809032:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574856.513020:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574856.513023:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574856.513026:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574856.832919:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574856.832926:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574856.832930:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574856.832938:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574856.832944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574856.832948:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574856.832959:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574856.832962:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574856.832964:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574856.832982:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574856.832986:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574856.832987:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574856.832989:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574856.832992:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574856.832994:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574856.832997:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574856.832998:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574856.833009:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574857.088930:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310216941, at 4310216960 00000100:00000010:0.0:1788574857.088946:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf240. 00000020:00000001:0.0:1788574857.537014:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574857.537017:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574857.537020:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574857.856930:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574857.856936:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574857.856940:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574857.856947:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574857.856953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574857.856959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574857.856969:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574857.856974:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574857.856979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.369143:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574858.369148:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574858.369152:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574858.369161:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516749952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.369180:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574858.369183:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574858.369188:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574858.369192:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574858.369199:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574858.369208:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516749952:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574858.369215:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574858.369220:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574858.369223:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574858.369225:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574858.369232:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574858.369235:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574858.369247:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574858.369251:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574858.369260:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574858.369276:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516749952, portal 25 00000100:00000001:0.0:1788574858.369279:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574858.369281:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788574858.369290:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516749952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.369314:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574858.369322:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574858.369325:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574858.369329:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516749952, offset 0 00000400:00000200:0.0:1788574858.369339:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574858.369350:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574858.369359:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574858.369367:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574858.369375:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574858.369385:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574858.369395:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574858.369405:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547511:547511:256:4294967295] 192.168.202.128@tcp LPNI seq info [547434:547434:8:4294967295] 00000400:00000200:0.0:1788574858.369419:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574858.369438:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574858.369451:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574858.369458:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574858.369468:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574858.369474:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574858.369514:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574858.369518:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574858.369527:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574858.369537:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574858.369549:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574858.369554:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574858.369559:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574858.369565:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516749952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.369588:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574858.369597:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516749952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.369613:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.369617:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574858.369625:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574858.369632:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574858.369636:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310222061 00000100:00000010:0.0:1788574858.369643:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574858.369653:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574858.369661:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310222061 00000100:00000010:0.0:1788574858.369667:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b505a0. 00000100:00000001:0.0:1788574858.369671:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574858.369674:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574858.369679:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.369682:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574858.369688:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574858/real 1788574858] req@ffff8fc4901a7800 x1875440516749952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574858.369702:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574858.369705:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574858.369708:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574858.369712:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574858.369715:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574858.369719:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574858.369726:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516749952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574858.369743:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516749952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.369771:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574858.369781:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.369785:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574858.369787:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574858.369795:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516749952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.369802:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574858.369828:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516749952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.369857:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574858.369862:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.369865:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574858.369867:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.369872:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574858.369877:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574858.369881:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574858.369889:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574858.369894:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574858.369903:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516749952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574858.369917:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516749952:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574858.369926:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.369931:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574858.369937:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516749952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.369950:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574858.369953:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574858.369956:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574858.369962:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574858.369965:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574858.369973:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574858.369977:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574858.369980:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788574858.369985:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574858.369988:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574858.369990:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788574858.369996:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574858.369998:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370002:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370008:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574858.370011:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.370015:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574858.370017:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574858.370020:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574858.370025:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516750080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370035:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574858.370037:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574858.370039:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574858.370041:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574858.370042:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574858.370045:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750080:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574858.370048:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574858.370050:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574858.370052:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574858.370054:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574858.370057:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574858.370059:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574858.370063:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574858.370065:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574858.370070:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574858.370073:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516750080, portal 10 00000100:00000001:0.0:1788574858.370076:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574858.370077:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788574858.370084:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516750080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370094:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574858.370096:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574858.370098:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574858.370100:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516750080, offset 0 00000400:00000200:0.0:1788574858.370103:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574858.370107:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574858.370111:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574858.370114:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574858.370115:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574858.370117:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574858.370119:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574858.370121:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547512:547512:256:4294967295] 192.168.202.128@tcp LPNI seq info [547435:547435:8:4294967295] 00000400:00000200:0.0:1788574858.370124:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574858.370129:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574858.370132:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574858.370134:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574858.370137:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574858.370139:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574858.370147:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574858.370148:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574858.370150:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574858.370153:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574858.370171:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574858.370172:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574858.370174:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574858.370177:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516750080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370183:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574858.370185:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516750080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370200:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.370202:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574858.370205:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574858.370208:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574858.370211:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574858.370213:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310218259 00000100:00000001:0.0:1788574858.370215:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310222061 00000100:00000001:0.0:1788574858.370216:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310218259 00000100:00000001:0.0:1788574858.370218:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310222061 00000100:00000001:0.0:1788574858.370219:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574858.370220:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.370222:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574858.370224:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574858/real 1788574858] req@ffff8fc4901a5c00 x1875440516750080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574858.370229:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574858.370230:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574858.370232:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574858.370233:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574858.370234:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574858.370236:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574858.370238:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516750080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574858.370244:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516750080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370247:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574858.370248:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370249:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574858.370250:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574858.370252:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516750080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370260:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574858.370264:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516750080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370269:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574858.370270:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370271:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574858.370273:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370274:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574858.370276:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574858.370278:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574858.370281:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574858.370282:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574858.370285:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516750080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574858.370290:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750080:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574858.370293:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574858.370295:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516750208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370299:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574858.370300:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574858.370301:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574858.370302:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574858.370304:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574858.370305:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750208:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574858.370307:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574858.370308:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574858.370313:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574858.370314:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574858.370316:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574858.370317:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574858.370320:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574858.370321:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574858.370323:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574858.370325:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516750208, portal 10 00000100:00000001:0.0:1788574858.370327:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574858.370328:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788574858.370332:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516750208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370336:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574858.370338:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574858.370339:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574858.370341:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516750208, offset 0 00000400:00000200:0.0:1788574858.370343:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574858.370346:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574858.370348:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574858.370350:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574858.370352:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574858.370353:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574858.370355:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574858.370357:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547513:547513:256:4294967295] 192.168.202.128@tcp LPNI seq info [547436:547436:8:4294967295] 00000400:00000200:0.0:1788574858.370360:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574858.370364:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574858.370367:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574858.370370:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574858.370374:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574858.370377:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574858.370388:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574858.370395:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574858.370398:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574858.370402:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574858.370406:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574858.370408:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574858.370411:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574858.370415:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516750208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370423:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574858.370426:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516750208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370431:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.370432:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574858.370435:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574858.370437:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574858.370440:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574858.370441:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310218266 00000100:00000001:0.0:1788574858.370443:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310222061 00000100:00000001:0.0:1788574858.370444:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310218266 00000100:00000001:0.0:1788574858.370446:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310222061 00000100:00000001:0.0:1788574858.370448:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574858.370449:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.370450:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574858.370453:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574858/real 1788574858] req@ffff8fc4901a5180 x1875440516750208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574858.370457:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574858.370458:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574858.370459:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574858.370461:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574858.370462:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574858.370468:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574858.370470:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516750208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574858.370476:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516750208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370480:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574858.370481:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370482:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574858.370483:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574858.370485:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516750208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370489:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574858.370492:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516750208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370496:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574858.370498:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370499:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574858.370499:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370501:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574858.370502:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574858.370504:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574858.370506:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574858.370507:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574858.370509:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516750208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574858.370514:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750208:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574858.370516:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574858.370521:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516750336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370526:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574858.370527:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574858.370528:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574858.370530:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574858.370531:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574858.370533:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750336:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574858.370535:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574858.370537:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574858.370538:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574858.370539:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574858.370541:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574858.370542:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574858.370545:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574858.370547:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574858.370549:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574858.370551:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516750336, portal 4 00000100:00000001:0.0:1788574858.370552:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574858.370553:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788574858.370557:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516750336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370561:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574858.370563:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574858.370564:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574858.370566:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516750336, offset 0 00000400:00000200:0.0:1788574858.370569:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574858.370571:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574858.370573:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574858.370575:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574858.370577:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574858.370578:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574858.370580:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574858.370586:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547514:547514:256:4294967295] 192.168.202.128@tcp LPNI seq info [547437:547437:8:4294967295] 00000400:00000200:0.0:1788574858.370588:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574858.370593:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574858.370596:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574858.370598:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574858.370601:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574858.370603:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574858.370610:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574858.370611:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574858.370613:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574858.370615:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574858.370617:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574858.370618:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574858.370620:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574858.370622:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516750336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370627:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574858.370630:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516750336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370634:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.370635:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574858.370636:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574858.370638:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574858.370640:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574858.370642:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310218406 00000100:00000001:0.0:1788574858.370643:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310222061 00000100:00000001:0.0:1788574858.370644:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310218406 00000100:00000001:0.0:1788574858.370646:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310222061 00000100:00000001:0.0:1788574858.370647:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574858.370653:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.370655:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574858.370657:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574858/real 1788574858] req@ffff8fc4901a7100 x1875440516750336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574858.370663:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574858.370664:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574858.370665:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574858.370667:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574858.370668:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574858.370669:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574858.370671:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516750336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574858.370676:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516750336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370680:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574858.370681:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370682:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574858.370683:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574858.370685:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516750336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370689:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574858.370691:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516750336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370695:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574858.370697:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370698:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574858.370699:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370700:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574858.370701:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574858.370702:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574858.370708:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574858.370709:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574858.370712:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516750336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574858.370716:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750336:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574858.370719:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370720:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574858.370722:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516750080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370726:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574858.370727:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574858.370728:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574858.370729:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574858.370731:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574858.370732:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574858.370733:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574858.370742:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788574858.370744:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574858.370745:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574858.370746:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788574858.370753:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574858.370753:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370755:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574858.370757:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516750208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370761:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574858.370762:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574858.370763:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574858.370765:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574858.370766:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574858.370767:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574858.370768:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574858.370772:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788574858.370774:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574858.370775:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574858.370776:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788574858.370777:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574858.370778:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370779:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574858.370782:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516750336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370785:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574858.370786:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574858.370787:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574858.370788:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574858.370789:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574858.370790:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574858.370791:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574858.370791:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788574858.370792:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574858.370793:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574858.370794:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788574858.370795:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574858.370796:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370797:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370800:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574858.370801:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.370802:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574858.370803:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574858.370804:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574858.370806:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516750464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370824:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574858.370825:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574858.370827:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574858.370828:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574858.370830:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574858.370832:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750464:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574858.370850:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574858.370852:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574858.370853:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574858.370854:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574858.370856:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574858.370858:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574858.370860:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574858.370862:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574858.370864:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574858.370867:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516750464, portal 4 00000100:00000001:0.0:1788574858.370868:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574858.370869:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788574858.370873:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516750464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370878:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574858.370879:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574858.370881:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574858.370882:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516750464, offset 0 00000400:00000200:0.0:1788574858.370885:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574858.370887:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574858.370890:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574858.370892:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574858.370893:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574858.370895:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574858.370897:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574858.370899:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547515:547515:256:4294967295] 192.168.202.128@tcp LPNI seq info [547438:547438:8:4294967295] 00000400:00000200:0.0:1788574858.370901:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574858.370906:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574858.370908:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574858.370910:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574858.370913:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574858.370918:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788574858.370926:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574858.370927:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574858.370929:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574858.370931:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574858.370934:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574858.370935:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574858.370937:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574858.370939:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516750464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370944:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574858.370946:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516750464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370950:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.370951:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574858.370953:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574858.370955:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574858.370958:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574858.370959:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310218409 00000100:00000001:0.0:1788574858.370961:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310222061 00000100:00000001:0.0:1788574858.370962:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310218409 00000100:00000001:0.0:1788574858.370964:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310222061 00000100:00000001:0.0:1788574858.370965:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574858.370966:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.370967:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574858.370969:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574858/real 1788574858] req@ffff8fc4901a4a80 x1875440516750464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574858.370974:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574858.370975:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574858.370980:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574858.370981:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574858.370982:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574858.370984:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574858.370986:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516750464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574858.370991:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516750464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.370996:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574858.370997:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.370998:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574858.370999:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574858.371001:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516750464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.371005:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574858.371007:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516750464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.371011:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574858.371012:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.371013:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574858.371014:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.371015:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574858.371016:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574858.371018:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574858.371020:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574858.371021:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574858.371024:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516750464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574858.371029:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750464:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574858.371034:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.371036:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574858.371038:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516750464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574858.371042:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574858.371043:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574858.371044:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574858.371046:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574858.371047:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574858.371047:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574858.371049:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574858.371050:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788574858.371051:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574858.371052:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574858.371054:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788574858.371057:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574858.371057:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.371058:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.371061:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574858.371062:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.371063:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574858.371064:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574858.880943:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574858.880952:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574858.880957:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574858.880965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574858.880972:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574858.880977:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.880992:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574858.880997:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574858.881001:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574859.584956:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574859.584959:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574859.584962:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574859.904930:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574859.904939:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574859.904963:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574859.904972:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574859.904977:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574859.904982:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574859.904997:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574859.905001:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574859.905005:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574860.929023:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574860.929028:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574860.929030:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574860.929034:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574860.929037:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574860.929042:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574860.929048:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574860.929051:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574860.929053:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574861.632981:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574861.632985:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574861.632988:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574861.952904:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574861.952911:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574861.952916:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574861.952925:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574861.952932:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574861.952936:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574861.952941:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574861.952945:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574861.952952:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574862.208957:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310222061, at 4310222080 00000100:00000010:0.0:1788574862.208981:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574862.657078:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574862.657085:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574862.657090:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574862.976898:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574862.976906:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574862.976910:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574862.976919:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574862.976941:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574862.976947:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574862.976963:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574862.976966:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574862.976969:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.489127:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574863.489135:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574863.489140:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574863.489153:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516750592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.489170:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574863.489173:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574863.489177:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574863.489181:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574863.489185:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574863.489193:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516750592:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574863.489200:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574863.489206:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574863.489209:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574863.489213:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574863.489219:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574863.489223:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574863.489231:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574863.489235:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574863.489242:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574863.489250:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516750592, portal 25 00000100:00000001:0.0:1788574863.489254:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574863.489258:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788574863.489271:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516750592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.489309:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574863.489317:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574863.489321:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574863.489325:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516750592, offset 0 00000400:00000200:0.0:1788574863.489347:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574863.489360:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574863.489373:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574863.489383:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574863.489392:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574863.489400:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574863.489407:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574863.489416:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547516:547516:256:4294967295] 192.168.202.128@tcp LPNI seq info [547439:547439:8:4294967295] 00000400:00000200:0.0:1788574863.489429:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574863.489446:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574863.489458:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574863.489464:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574863.489475:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574863.489481:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788574863.489504:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574863.489507:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574863.489515:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574863.489522:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574863.489529:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574863.489534:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574863.489539:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574863.489548:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516750592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.489563:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574863.489573:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516750592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.489590:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.489595:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574863.489614:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574863.489627:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574863.489631:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310227181 00000100:00000010:0.0:1788574863.489646:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574863.489656:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574863.489674:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310227181 00000100:00000010:0.0:1788574863.489683:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50b40. 00000100:00000001:0.0:1788574863.489690:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574863.489698:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574863.489706:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.489724:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574863.489734:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574863/real 1788574863] req@ffff8fc4ac822d80 x1875440516750592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574863.489755:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574863.489758:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574863.489763:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574863.489769:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574863.489774:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574863.489781:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574863.489790:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516750592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574863.489836:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516750592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.489845:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574863.489848:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.489853:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574863.489856:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574863.489862:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516750592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.489877:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574863.489889:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516750592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.489909:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574863.489924:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.489929:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574863.489933:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.489940:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574863.489945:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574863.489953:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574863.489962:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574863.489969:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574863.489995:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516750592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574863.490010:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516750592:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574863.490020:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.490026:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574863.490035:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516750592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490049:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574863.490054:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574863.490059:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574863.490067:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574863.490071:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574863.490074:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574863.490080:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574863.490085:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788574863.490092:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574863.490095:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574863.490099:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788574863.490105:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574863.490109:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.490115:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.490124:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574863.490127:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.490130:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574863.490139:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574863.490140:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574863.490145:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516750720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490152:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574863.490153:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574863.490156:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574863.490158:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574863.490160:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574863.490163:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574863.490166:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574863.490168:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574863.490170:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574863.490172:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574863.490175:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574863.490177:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574863.490181:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574863.490184:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574863.490188:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574863.490191:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516750720, portal 10 00000100:00000001:0.0:1788574863.490193:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574863.490195:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788574863.490201:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516750720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490207:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574863.490210:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574863.490212:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574863.490214:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516750720, offset 0 00000400:00000200:0.0:1788574863.490220:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574863.490224:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574863.490228:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574863.490232:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574863.490234:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574863.490244:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574863.490246:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574863.490249:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547517:547517:256:4294967295] 192.168.202.128@tcp LPNI seq info [547440:547440:8:4294967295] 00000400:00000200:0.0:1788574863.490254:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574863.490261:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574863.490267:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574863.490271:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574863.490276:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574863.490279:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788574863.490291:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574863.490293:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574863.490296:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574863.490300:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574863.490303:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574863.490305:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574863.490309:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574863.490313:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516750720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490322:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574863.490325:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516750720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490331:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.490333:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574863.490337:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574863.490341:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574863.490345:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574863.490347:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310223379 00000100:00000001:0.0:1788574863.490349:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310227181 00000100:00000001:0.0:1788574863.490351:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310223379 00000100:00000001:0.0:1788574863.490359:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310227181 00000100:00000001:0.0:1788574863.490362:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574863.490364:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.490366:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574863.490371:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574863/real 1788574863] req@ffff8fc4ac821f80 x1875440516750720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574863.490381:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574863.490382:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574863.490384:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574863.490387:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574863.490389:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574863.490391:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574863.490395:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516750720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574863.490405:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516750720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490412:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574863.490414:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.490416:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574863.490418:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574863.490422:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516750720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490430:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574863.490434:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516750720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490442:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574863.490444:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.490446:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574863.490447:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.490456:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574863.490459:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574863.490462:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574863.490466:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574863.490469:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574863.490473:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516750720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574863.490482:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574863.490487:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574863.490491:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516750848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490498:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574863.490499:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574863.490501:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574863.490504:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574863.490506:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574863.490510:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750848:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574863.490513:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574863.490516:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574863.490517:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574863.490519:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574863.490524:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574863.490526:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574863.490530:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574863.490533:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574863.490537:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574863.490540:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516750848, portal 10 00000100:00000001:0.0:1788574863.490542:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574863.490544:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788574863.490550:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516750848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490564:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574863.490567:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574863.490570:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574863.490573:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516750848, offset 0 00000400:00000200:0.0:1788574863.490577:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574863.490581:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574863.490586:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574863.490589:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574863.490591:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574863.490594:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574863.490596:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574863.490599:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547518:547518:256:4294967295] 192.168.202.128@tcp LPNI seq info [547441:547441:8:4294967295] 00000400:00000200:0.0:1788574863.490604:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574863.490611:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574863.490616:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574863.490620:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574863.490631:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574863.490635:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788574863.490647:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574863.490649:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574863.490652:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574863.490656:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574863.490660:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574863.490662:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574863.490664:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574863.490668:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516750848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490676:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574863.490679:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516750848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490697:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.490699:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574863.490704:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574863.490720:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574863.490725:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574863.490728:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310223386 00000100:00000001:0.0:1788574863.490730:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310227181 00000100:00000001:0.0:1788574863.490732:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310223386 00000100:00000001:0.0:1788574863.490734:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310227181 00000100:00000001:0.0:1788574863.490737:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574863.490739:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.490741:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574863.490746:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574863/real 1788574863] req@ffff8fc4ac823480 x1875440516750848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574863.490755:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574863.490757:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574863.490759:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574863.490762:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574863.490764:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574863.490767:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574863.490770:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516750848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574863.490778:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516750848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490785:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574863.490787:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.490789:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574863.490790:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574863.490793:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516750848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490840:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574863.490846:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516750848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490854:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574863.490857:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.490859:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574863.490860:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.490862:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574863.490864:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574863.490867:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574863.490870:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574863.490873:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574863.490877:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516750848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574863.490885:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750848:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574863.490890:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574863.490895:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516750976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490902:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574863.490904:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574863.490907:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574863.490909:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574863.490911:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574863.490915:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574863.490918:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574863.490921:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574863.490930:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574863.490932:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574863.490936:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574863.490938:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574863.490942:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574863.490945:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574863.490948:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574863.490951:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516750976, portal 4 00000100:00000001:0.0:1788574863.490953:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574863.490955:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788574863.490961:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516750976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.490968:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574863.490971:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574863.490973:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574863.490976:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516750976, offset 0 00000400:00000200:0.0:1788574863.490980:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574863.490985:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574863.490989:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574863.490993:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574863.490996:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574863.490999:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574863.491002:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574863.491005:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547519:547519:256:4294967295] 192.168.202.128@tcp LPNI seq info [547442:547442:8:4294967295] 00000400:00000200:0.0:1788574863.491009:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574863.491015:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574863.491020:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574863.491023:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574863.491028:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574863.491031:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788574863.491043:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574863.491052:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574863.491057:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574863.491062:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574863.491067:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574863.491069:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574863.491072:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574863.491076:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516750976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491084:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574863.491086:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516750976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491092:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.491094:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574863.491097:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574863.491100:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574863.491103:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574863.491105:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310223527 00000100:00000001:0.0:1788574863.491107:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310227181 00000100:00000001:0.0:1788574863.491109:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310223527 00000100:00000001:0.0:1788574863.491111:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310227181 00000100:00000001:0.0:1788574863.491113:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574863.491115:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.491117:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574863.491120:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574863/real 1788574863] req@ffff8fc4ac822300 x1875440516750976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574863.491128:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574863.491129:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574863.491131:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574863.491133:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574863.491134:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574863.491143:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574863.491146:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516750976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574863.491154:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516750976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491160:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574863.491161:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.491163:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574863.491165:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574863.491168:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516750976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491174:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574863.491178:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516750976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491184:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574863.491187:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.491189:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574863.491190:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.491192:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574863.491194:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574863.491197:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574863.491201:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574863.491203:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574863.491207:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516750976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574863.491215:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516750976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574863.491219:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574863.491223:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516751104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491244:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574863.491246:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574863.491248:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574863.491250:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574863.491252:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574863.491256:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516751104:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574863.491259:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574863.491261:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574863.491263:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574863.491265:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574863.491269:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574863.491271:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574863.491275:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574863.491277:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574863.491281:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574863.491284:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516751104, portal 4 00000100:00000001:0.0:1788574863.491287:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574863.491288:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788574863.491295:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516751104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491303:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574863.491305:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574863.491308:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574863.491311:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516751104, offset 0 00000400:00000200:0.0:1788574863.491315:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574863.491319:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574863.491323:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574863.491326:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574863.491328:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574863.491331:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574863.491333:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574863.491343:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547520:547520:256:4294967295] 192.168.202.128@tcp LPNI seq info [547443:547443:8:4294967295] 00000400:00000200:0.0:1788574863.491348:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574863.491356:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574863.491362:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574863.491365:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574863.491370:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574863.491372:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788574863.491383:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574863.491385:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574863.491388:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574863.491392:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574863.491395:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574863.491397:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574863.491400:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574863.491404:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516751104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491413:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574863.491416:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516751104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491424:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.491425:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574863.491429:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574863.491432:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574863.491436:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574863.491438:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310223529 00000100:00000001:0.0:1788574863.491440:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310227181 00000100:00000001:0.0:1788574863.491442:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310223529 00000100:00000001:0.0:1788574863.491444:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310227181 00000100:00000001:0.0:1788574863.491446:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574863.491455:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.491457:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574863.491462:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574863/real 1788574863] req@ffff8fc4ac820700 x1875440516751104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574863.491472:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574863.491473:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574863.491475:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574863.491478:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574863.491479:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574863.491482:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574863.491485:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516751104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574863.491492:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516751104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491498:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574863.491500:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.491502:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574863.491503:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574863.491507:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516751104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491514:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574863.491518:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516751104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491525:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574863.491528:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.491529:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574863.491531:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.491533:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574863.491535:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574863.491537:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574863.491562:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574863.491565:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574863.491570:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516751104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574863.491576:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516751104:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574863.491579:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.491580:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574863.491582:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516750720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491587:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574863.491588:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574863.491590:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788574863.491593:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574863.491594:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574863.491595:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574863.491597:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574863.491599:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788574863.491601:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574863.491603:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574863.491604:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788574863.491606:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574863.491607:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.491608:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574863.491610:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516750848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491614:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574863.491615:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574863.491616:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574863.491618:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574863.491619:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574863.491619:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574863.491620:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574863.491625:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788574863.491627:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574863.491628:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574863.491629:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788574863.491631:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574863.491632:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.491633:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574863.491635:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516750976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491638:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574863.491639:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574863.491640:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574863.491641:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574863.491642:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574863.491643:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574863.491644:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574863.491645:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788574863.491646:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574863.491647:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574863.491648:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788574863.491649:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574863.491651:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.491652:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574863.491655:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516751104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574863.491668:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574863.491670:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574863.491671:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574863.491674:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574863.491675:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574863.491677:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574863.491679:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574863.491680:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862ac00. 02000000:00000001:0.0:1788574863.491683:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574863.491684:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574863.491686:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788574863.491694:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574863.491696:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.491698:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574863.491704:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574863.491706:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574863.491720:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574863.491722:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574864.000910:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574864.000918:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574864.000921:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574864.000928:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574864.000935:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574864.000939:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574864.000943:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574864.000948:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574864.000954:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574864.000984:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574864.000989:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574864.000991:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574864.000994:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574864.000997:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574864.001000:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574864.001003:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574864.001005:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574864.001007:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574864.705005:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574864.705008:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574864.705011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574866.048970:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574866.048977:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574866.048985:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574866.049001:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574866.049008:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574866.049013:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574866.049024:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574866.049027:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574866.049031:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574866.753020:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574866.753041:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574866.753044:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574867.072884:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574867.072890:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574867.072895:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574867.072904:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574867.072913:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574867.072919:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574867.072936:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574867.072941:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574867.072947:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574867.328909:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310227181, at 4310227200 00000100:00000010:0.0:1788574867.328922:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574867.776954:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574867.776958:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574867.776961:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574868.096905:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574868.096914:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574868.096917:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574868.096922:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574868.096926:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574868.096930:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.096934:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574868.096937:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.096941:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.097030:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574868.097033:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574868.097036:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574868.097043:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574868.097049:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574868.097052:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.097054:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574868.097056:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.097057:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.609161:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574868.609166:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574868.609171:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574868.609187:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7100 x1875440516751232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.609208:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574868.609211:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574868.609216:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574868.609220:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574868.609225:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574868.609233:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516751232:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574868.609239:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574868.609244:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574868.609247:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574868.609251:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574868.609256:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574868.609262:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574868.609274:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574868.609280:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574868.609292:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574868.609302:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516751232, portal 25 00000100:00000001:0.0:1788574868.609315:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574868.609319:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605333248 : -123400104218368 : ffff8fc4ac9f7100) 00000100:00000040:0.0:1788574868.609327:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7100 x1875440516751232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.609337:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574868.609343:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574868.609347:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574868.609351:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516751232, offset 0 00000400:00000200:0.0:1788574868.609358:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574868.609367:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574868.609377:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574868.609385:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574868.609391:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574868.609399:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574868.609403:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574868.609417:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547521:547521:256:4294967295] 192.168.202.128@tcp LPNI seq info [547444:547444:8:4294967295] 00000400:00000200:0.0:1788574868.609426:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574868.609435:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574868.609442:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574868.609448:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574868.609459:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574868.609466:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788574868.609489:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574868.609491:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574868.609498:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574868.609505:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574868.609510:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574868.609513:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574868.609519:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574868.609528:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7100 x1875440516751232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.609544:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574868.609551:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7100 x1875440516751232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.609568:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.609572:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574868.609578:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574868.609600:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574868.609602:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310232301 00000100:00000010:0.0:1788574868.609608:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574868.609618:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574868.609624:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310232301 00000100:00000010:0.0:1788574868.609631:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50b40. 00000100:00000001:0.0:1788574868.609637:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574868.609642:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574868.609654:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.609660:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574868.609667:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574868/real 1788574868] req@ffff8fc4ac9f7100 x1875440516751232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574868.609685:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574868.609688:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574868.609693:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574868.609697:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574868.609699:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574868.609704:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574868.609712:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7100 x1875440516751232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574868.609736:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7100 x1875440516751232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.609748:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574868.609751:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.609754:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574868.609758:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574868.609764:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7100 x1875440516751232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.609777:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574868.609786:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7100 x1875440516751232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.609797:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574868.609802:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.609806:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574868.609827:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.609831:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574868.609833:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574868.609836:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574868.609842:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574868.609856:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574868.609867:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7100 x1875440516751232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574868.609886:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516751232:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574868.609894:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.609900:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574868.609908:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7100 x1875440516751232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.609926:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574868.609930:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574868.609934:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574868.609941:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574868.609944:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574868.609947:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574868.609951:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574868.609955:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36c00. 02000000:00000001:0.0:1788574868.609962:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574868.609965:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574868.609968:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7100. 00000100:00000001:0.0:1788574868.609973:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574868.609976:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.609981:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.609988:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574868.609991:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.609996:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574868.609999:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574868.610002:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574868.610009:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4000 x1875440516751360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610028:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574868.610032:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574868.610036:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574868.610039:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574868.610052:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574868.610061:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516751360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574868.610068:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574868.610072:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574868.610074:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574868.610079:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574868.610085:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574868.610090:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574868.610098:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574868.610103:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574868.610109:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574868.610113:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516751360, portal 10 00000100:00000001:0.0:1788574868.610117:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574868.610120:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605320704 : -123400104230912 : ffff8fc4ac9f4000) 00000100:00000040:0.0:1788574868.610130:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4000 x1875440516751360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610141:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574868.610147:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574868.610152:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574868.610156:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516751360, offset 0 00000400:00000200:0.0:1788574868.610161:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574868.610167:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574868.610174:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574868.610177:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574868.610183:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574868.610188:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574868.610193:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574868.610198:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547522:547522:256:4294967295] 192.168.202.128@tcp LPNI seq info [547445:547445:8:4294967295] 00000400:00000200:0.0:1788574868.610205:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574868.610211:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574868.610215:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574868.610222:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574868.610226:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574868.610228:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788574868.610236:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574868.610238:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574868.610240:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574868.610242:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574868.610245:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574868.610246:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574868.610248:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574868.610251:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4000 x1875440516751360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610257:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574868.610259:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4000 x1875440516751360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610263:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.610264:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574868.610267:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574868.610271:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574868.610274:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574868.610276:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310228499 00000100:00000001:0.0:1788574868.610279:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310232301 00000100:00000001:0.0:1788574868.610282:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310228499 00000100:00000001:0.0:1788574868.610284:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310232301 00000100:00000001:0.0:1788574868.610286:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574868.610288:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.610290:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574868.610294:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574868/real 1788574868] req@ffff8fc4ac9f4000 x1875440516751360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574868.610317:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574868.610318:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574868.610320:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574868.610323:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574868.610325:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574868.610327:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574868.610330:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4000 x1875440516751360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574868.610336:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4000 x1875440516751360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610341:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574868.610342:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610343:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574868.610344:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574868.610346:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4000 x1875440516751360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610350:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574868.610353:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4000 x1875440516751360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610358:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574868.610360:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610361:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574868.610361:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610363:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574868.610364:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574868.610366:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574868.610368:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574868.610369:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574868.610376:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4000 x1875440516751360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574868.610381:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516751360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574868.610384:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574868.610386:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6680 x1875440516751488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610390:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574868.610391:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574868.610392:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574868.610394:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574868.610395:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574868.610397:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516751488:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574868.610399:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574868.610400:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574868.610401:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574868.610402:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574868.610404:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574868.610406:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574868.610409:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574868.610410:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574868.610413:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574868.610415:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516751488, portal 10 00000100:00000001:0.0:1788574868.610416:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574868.610417:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605330560 : -123400104221056 : ffff8fc4ac9f6680) 00000100:00000040:0.0:1788574868.610421:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6680 x1875440516751488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610426:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574868.610427:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574868.610429:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574868.610430:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516751488, offset 0 00000400:00000200:0.0:1788574868.610436:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574868.610439:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574868.610442:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574868.610444:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574868.610445:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574868.610447:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574868.610449:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574868.610451:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547523:547523:256:4294967295] 192.168.202.128@tcp LPNI seq info [547446:547446:8:4294967295] 00000400:00000200:0.0:1788574868.610454:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574868.610458:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574868.610461:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574868.610463:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574868.610466:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574868.610467:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788574868.610479:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574868.610480:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574868.610483:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574868.610485:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574868.610487:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574868.610488:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574868.610490:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574868.610493:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6680 x1875440516751488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610498:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574868.610500:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6680 x1875440516751488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610504:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.610505:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574868.610507:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574868.610509:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574868.610512:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574868.610516:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310228506 00000100:00000001:0.0:1788574868.610518:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310232301 00000100:00000001:0.0:1788574868.610519:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310228506 00000100:00000001:0.0:1788574868.610521:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310232301 00000100:00000001:0.0:1788574868.610522:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574868.610524:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.610525:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574868.610528:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574868/real 1788574868] req@ffff8fc4ac9f6680 x1875440516751488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574868.610532:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574868.610534:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574868.610535:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574868.610536:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574868.610537:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574868.610539:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574868.610541:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6680 x1875440516751488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574868.610547:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6680 x1875440516751488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610550:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574868.610551:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610552:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574868.610553:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574868.610555:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6680 x1875440516751488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610559:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574868.610562:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6680 x1875440516751488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610566:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574868.610570:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610571:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574868.610572:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610574:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574868.610576:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574868.610578:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574868.610580:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574868.610596:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574868.610599:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6680 x1875440516751488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574868.610605:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516751488:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574868.610608:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574868.610610:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7480 x1875440516751616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610614:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574868.610615:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574868.610617:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574868.610618:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574868.610619:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574868.610621:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516751616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574868.610623:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574868.610624:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574868.610625:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574868.610626:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574868.610629:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574868.610630:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574868.610633:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574868.610634:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574868.610637:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574868.610639:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516751616, portal 4 00000100:00000001:0.0:1788574868.610644:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574868.610645:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605334144 : -123400104217472 : ffff8fc4ac9f7480) 00000100:00000040:0.0:1788574868.610649:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7480 x1875440516751616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610654:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574868.610655:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574868.610657:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574868.610659:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516751616, offset 0 00000400:00000200:0.0:1788574868.610661:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574868.610664:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574868.610666:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574868.610668:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574868.610670:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574868.610672:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574868.610674:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574868.610676:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547524:547524:256:4294967295] 192.168.202.128@tcp LPNI seq info [547447:547447:8:4294967295] 00000400:00000200:0.0:1788574868.610678:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574868.610683:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574868.610686:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574868.610688:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574868.610690:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574868.610692:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788574868.610699:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574868.610700:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574868.610702:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574868.610705:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574868.610707:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574868.610708:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574868.610710:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574868.610712:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7480 x1875440516751616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610725:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574868.610727:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7480 x1875440516751616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610732:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.610733:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574868.610735:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574868.610738:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574868.610740:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574868.610741:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310228646 00000100:00000001:0.0:1788574868.610743:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310232301 00000100:00000001:0.0:1788574868.610744:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310228646 00000100:00000001:0.0:1788574868.610746:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310232301 00000100:00000001:0.0:1788574868.610747:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574868.610748:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.610750:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574868.610752:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574868/real 1788574868] req@ffff8fc4ac9f7480 x1875440516751616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574868.610757:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574868.610757:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574868.610759:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574868.610760:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574868.610761:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574868.610763:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574868.610765:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7480 x1875440516751616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574868.610770:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7480 x1875440516751616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610775:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574868.610780:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610782:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574868.610784:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574868.610788:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7480 x1875440516751616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610795:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574868.610799:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7480 x1875440516751616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610827:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574868.610831:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610833:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574868.610835:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610836:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574868.610838:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574868.610841:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574868.610845:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574868.610848:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574868.610853:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7480 x1875440516751616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574868.610861:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516751616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574868.610865:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610867:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574868.610871:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4000 x1875440516751360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610878:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574868.610879:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574868.610882:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788574868.610886:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574868.610888:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574868.610895:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574868.610898:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574868.610900:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35c00. 02000000:00000001:0.0:1788574868.610902:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574868.610904:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574868.610906:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4000. 00000100:00000001:0.0:1788574868.610909:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574868.610910:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610912:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574868.610917:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6680 x1875440516751488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610923:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574868.610925:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574868.610926:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788574868.610929:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574868.610930:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574868.610931:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574868.610933:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574868.610934:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35000. 02000000:00000001:0.0:1788574868.610936:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574868.610938:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574868.610939:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6680. 00000100:00000001:0.0:1788574868.610941:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574868.610943:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610944:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574868.610948:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7480 x1875440516751616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.610954:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574868.610955:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574868.610957:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574868.610959:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574868.610961:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574868.610962:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574868.610963:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574868.610965:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788574868.610967:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574868.610974:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574868.610976:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7480. 00000100:00000001:0.0:1788574868.610978:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574868.610980:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610981:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.610986:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574868.610987:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.610989:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574868.610990:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574868.610992:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574868.610996:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4e00 x1875440516751744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.611003:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574868.611004:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574868.611006:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574868.611008:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574868.611010:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574868.611013:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516751744:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574868.611016:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574868.611018:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574868.611020:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574868.611022:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574868.611024:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574868.611026:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574868.611030:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574868.611032:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574868.611036:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574868.611039:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516751744, portal 4 00000100:00000001:0.0:1788574868.611041:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574868.611042:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605324288 : -123400104227328 : ffff8fc4ac9f4e00) 00000100:00000040:0.0:1788574868.611048:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4e00 x1875440516751744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.611054:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574868.611056:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574868.611063:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574868.611065:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516751744, offset 0 00000400:00000200:0.0:1788574868.611070:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574868.611074:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574868.611077:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574868.611080:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574868.611083:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574868.611086:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574868.611088:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574868.611091:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547525:547525:256:4294967295] 192.168.202.128@tcp LPNI seq info [547448:547448:8:4294967295] 00000400:00000200:0.0:1788574868.611095:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574868.611102:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574868.611106:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574868.611109:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574868.611113:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574868.611116:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788574868.611127:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574868.611129:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574868.611131:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574868.611135:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574868.611138:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574868.611140:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574868.611143:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574868.611146:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4e00 x1875440516751744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.611154:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574868.611157:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4e00 x1875440516751744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.611163:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.611165:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574868.611174:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574868.611177:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574868.611181:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574868.611183:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310228649 00000100:00000001:0.0:1788574868.611186:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310232301 00000100:00000001:0.0:1788574868.611188:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310228649 00000100:00000001:0.0:1788574868.611189:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310232301 00000100:00000001:0.0:1788574868.611191:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574868.611192:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.611194:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574868.611197:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574868/real 1788574868] req@ffff8fc4ac9f4e00 x1875440516751744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574868.611202:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574868.611203:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574868.611204:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574868.611206:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574868.611207:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574868.611209:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574868.611211:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4e00 x1875440516751744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574868.611216:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4e00 x1875440516751744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.611221:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574868.611222:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.611223:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574868.611224:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574868.611226:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4e00 x1875440516751744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.611230:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574868.611232:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4e00 x1875440516751744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.611241:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574868.611242:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.611244:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574868.611244:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.611246:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574868.611247:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574868.611248:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574868.611250:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574868.611252:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574868.611254:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4e00 x1875440516751744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574868.611259:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516751744:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574868.611262:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.611263:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574868.611265:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4e00 x1875440516751744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574868.611271:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574868.611272:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574868.611274:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788574868.611277:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574868.611278:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574868.611280:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574868.611282:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574868.611283:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34c00. 02000000:00000001:0.0:1788574868.611286:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574868.611287:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574868.611289:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4e00. 00000100:00000001:0.0:1788574868.611291:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574868.611293:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.611300:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574868.611304:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574868.611305:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574868.611307:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574868.611309:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574869.120930:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574869.120936:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574869.120941:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574869.120948:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574869.120956:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574869.120963:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574869.120971:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574869.120976:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574869.120983:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574869.824986:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574869.824990:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574869.824994:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574870.144919:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574870.144930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574870.144934:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574870.144940:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574870.144960:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574870.144966:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574870.144973:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574870.144979:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574870.144984:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574870.145045:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574870.145049:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574870.145050:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574870.145053:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574870.145056:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574870.145058:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574870.145060:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574870.145061:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574870.145063:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574871.872956:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574871.872959:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574871.872961:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574872.192939:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574872.192958:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574872.192963:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574872.192970:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574872.192977:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574872.192983:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574872.192993:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574872.192996:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574872.192999:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574872.448921:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310232301, at 4310232320 00000100:00000010:0.0:1788574872.448944:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574872.897090:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574872.897095:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574872.897099:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574873.216957:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574873.216969:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574873.216973:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574873.216983:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574873.216991:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574873.216998:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574873.217013:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574873.217017:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574873.217022:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.217059:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574873.217063:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574873.217065:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574873.217068:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574873.217071:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574873.217074:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574873.217076:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574873.217078:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574873.217081:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574873.728957:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574873.728973:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574873.728976:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574873.728981:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574873.729009:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574873.729017:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15561 00000400:00000200:0.0:1788574873.729099:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574873.729106:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574873.729109:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574873.729112:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729114:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574873.729116:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729118:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574873.729122:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574873.729124:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574873.729126:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574873.729129:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574873.729131:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574873.729133:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574873.729134:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729135:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574873.729137:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729138:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574873.729140:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574873.729142:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729143:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574873.729145:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574873.729148:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574873.729150:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729152:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574873.729154:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574873.729162:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574873.729164:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574873.729168:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574873.729169:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729178:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574873.729181:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574873.729186:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574873.729194:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574873.729201:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574873.729210:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516751872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574873.729221:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574873.729226:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574873.729232:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516751872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574873.729255:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574873.729258:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574873.729261:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574873.729262:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574873.729264:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574873.729266:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574873.729268:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15561 00000400:00000200:0.0:1788574873.729271:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574873.729275:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574873.729277:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574873.729279:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729281:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574873.729282:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729284:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574873.729286:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574873.729287:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574873.729291:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574873.729293:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574873.729297:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574873.729299:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574873.729308:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729310:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574873.729313:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729317:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574873.729321:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574873.729325:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729329:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574873.729331:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574873.729337:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574873.729339:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729340:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574873.729344:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574873.729348:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729351:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574873.729354:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574873.729357:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574873.729359:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574873.729362:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729364:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574873.729367:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574873.729369:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574873.729374:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574873.729379:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574873.729384:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516752000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574873.729391:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574873.729393:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574873.729395:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516752000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574873.729403:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574873.729407:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574873.729415:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574873.729417:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574873.729420:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574873.729426:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574873.729427:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15561 00000400:00000200:0.0:1788574873.729434:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574873.729436:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574873.729439:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574873.729442:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729446:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574873.729448:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729450:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574873.729451:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574873.729452:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574873.729453:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574873.729526:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574873.729530:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574873.729532:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574873.729539:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729541:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574873.729543:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729545:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574873.729548:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574873.729552:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729556:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574873.729557:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574873.729559:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574873.729563:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729566:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574873.729568:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574873.729575:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729578:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574873.729582:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574873.729593:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574873.729599:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574873.729602:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729605:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574873.729608:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574873.729611:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574873.729617:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574873.729622:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574873.729630:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516752128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574873.729640:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574873.729643:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574873.729648:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516752128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574873.729657:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574873.729662:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574873.729664:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574873.729667:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574873.729668:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574873.729672:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574873.729676:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15561 00000400:00000200:0.0:1788574873.729681:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574873.729685:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574873.729689:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574873.729692:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729695:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574873.729699:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729707:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574873.729709:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574873.729710:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574873.729714:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574873.729716:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574873.729721:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574873.729725:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574873.729728:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729730:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574873.729731:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729735:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574873.729737:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574873.729741:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729743:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574873.729744:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574873.729746:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574873.729749:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729751:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574873.729754:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574873.729758:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729761:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574873.729765:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574873.729768:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574873.729771:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574873.729772:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729774:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574873.729776:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574873.729777:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574873.729781:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574873.729786:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574873.729793:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516752256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574873.729827:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574873.729828:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574873.729832:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516752256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574873.729840:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574873.729845:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574873.729850:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574873.729851:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574873.729853:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574873.729857:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574873.729860:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15561 00000400:00000200:0.0:1788574873.729865:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574873.729868:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574873.729870:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574873.729873:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729876:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574873.729877:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729879:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574873.729881:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574873.729883:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574873.729886:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574873.729888:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574873.729891:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574873.729893:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574873.729896:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729900:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574873.729902:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729904:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574873.729912:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574873.729916:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574873.729919:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574873.729921:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574873.729924:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574873.729928:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729931:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574873.729934:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574873.729938:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729939:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574873.729941:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574873.729943:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574873.729946:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574873.729947:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574873.729950:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574873.729952:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574873.729955:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574873.729962:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574873.729968:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574873.729977:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516752384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574873.729991:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574873.729995:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574873.730003:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516752384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574873.730023:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574873.730030:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15571) 00000100:00000001:0.0:1788574874.241002:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574874.241008:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574874.241010:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574874.241014:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574874.241018:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574874.241043:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574874.241049:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574874.241055:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574874.241058:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574874.945045:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574874.945048:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574874.945051:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574876.289009:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574876.289024:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574876.289029:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574876.289041:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574876.289053:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574876.289067:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574876.289078:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574876.289087:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574876.289098:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574876.993011:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574876.993017:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574876.993022:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574877.568903:0:265819:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310237422, at 4310237440 00000100:00000010:0.0:1788574877.568917:0:265819:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf480. 00000020:00000001:0.0:1788574878.017013:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574878.017016:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574878.017019:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574878.336924:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574878.336929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574878.336931:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574878.336937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574878.336940:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574878.336943:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574878.336949:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574878.336953:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574878.336956:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574878.848943:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788574878.848957:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788574878.848964:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1788574878.849144:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788574878.849151:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574878.849168:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574878.849176:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574878.849182:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574879.360912:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574879.360920:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574879.360924:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574879.360932:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574879.360939:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574879.360946:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574879.360971:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574879.360975:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574879.360978:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574880.065049:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574880.065055:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574880.065059:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574880.384929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574880.384937:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574880.384940:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574880.384948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574880.384953:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574880.384957:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574880.384960:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574880.384964:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574880.384969:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574881.408948:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574881.408955:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574881.408957:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574881.408965:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574881.408972:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574881.408978:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574881.408995:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574881.408998:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574881.409002:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574881.409019:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574881.409023:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574881.409024:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574881.409049:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574881.409052:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574881.409054:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574881.409055:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574881.409056:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574881.409058:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574882.113078:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574882.113082:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574882.113085:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574882.432957:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574882.432963:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574882.432966:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574882.432972:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574882.432977:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574882.432980:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574882.432983:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574882.432985:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574882.432990:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574882.688946:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310242541, at 4310242560 00000100:00000010:0.0:1788574882.688971:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf660. 00000020:00000001:0.0:1788574883.137048:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574883.137053:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574883.137056:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574883.456923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574883.456930:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574883.456935:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574883.456943:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574883.456950:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574883.456954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574883.456967:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574883.456970:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574883.456973:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574885.185021:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574885.185026:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574885.185030:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574885.505070:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574885.505076:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574885.505096:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574885.505101:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574885.505104:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574885.505108:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574885.505112:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574885.505113:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574885.505119:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574887.232944:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574887.232948:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574887.232950:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574887.553063:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574887.553067:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574887.553075:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574887.553079:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574887.553081:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574887.553084:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574887.554384:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574887.554387:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574887.554391:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574887.808912:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310247662, at 4310247680 00000100:00000010:0.0:1788574887.808933:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf0c0. 00000020:00000001:0.0:1788574888.257059:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574888.257064:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574888.257067:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574888.576922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574888.576928:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574888.576932:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574888.576939:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574888.576947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574888.576952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574888.576967:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574888.576970:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574888.576973:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574889.600937:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574889.600948:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574889.600957:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574889.600964:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574889.600986:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574889.600990:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574889.600995:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574889.600997:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574889.601000:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574890.305005:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574890.305009:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574890.305011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574890.624930:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574890.624935:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574890.624940:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574890.624946:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574890.624950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574890.624957:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574890.624972:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574890.624976:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574890.624979:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574890.624993:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574890.624997:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574890.624998:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574890.625000:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574890.625001:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574890.625003:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574890.625020:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574890.625024:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574890.625027:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574891.648945:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574891.648960:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574891.648963:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574891.648968:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574891.648972:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574891.648976:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574891.648993:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574891.648996:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574891.648999:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574892.352996:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574892.353001:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574892.353005:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574892.673006:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574892.673013:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574892.673017:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574892.673024:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574892.673032:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574892.673038:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574892.673041:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574892.673042:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574892.673044:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574892.928896:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310252781, at 4310252800 00000100:00000010:0.0:1788574892.928922:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf840. 00000020:00000001:0.0:1788574893.377027:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574893.377030:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574893.377033:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574893.696924:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574893.696933:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574893.696938:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574893.696948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574893.696957:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574893.696965:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574893.697260:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574893.697263:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574893.697266:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.209144:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574894.209151:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574894.209156:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574894.209169:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516754432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.209197:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574894.209200:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574894.209208:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574894.209214:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574894.209222:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574894.209263:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516754432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574894.209275:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574894.209289:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574894.209293:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574894.209299:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574894.209305:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574894.209309:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574894.209316:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574894.209323:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574894.209335:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574894.209347:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516754432, portal 25 00000100:00000001:0.0:1788574894.209353:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574894.209356:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788574894.209371:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516754432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.209397:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574894.209406:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574894.209412:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574894.209417:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516754432, offset 0 00000400:00000200:0.0:1788574894.209431:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574894.209445:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574894.209456:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574894.209465:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574894.209473:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574894.209480:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574894.209487:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574894.209497:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547546:547546:256:4294967295] 192.168.202.128@tcp LPNI seq info [547469:547469:8:4294967295] 00000400:00000200:0.0:1788574894.209510:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574894.209527:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574894.209539:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574894.209549:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574894.209560:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574894.209567:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574894.209593:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574894.209606:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574894.209616:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574894.209627:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574894.209635:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574894.209639:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574894.209647:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574894.209655:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516754432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.209678:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574894.209688:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516754432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.209710:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.209714:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574894.209722:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574894.209729:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574894.209735:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310257901 00000100:00000010:0.0:1788574894.209744:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574894.209757:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574894.209766:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310257901 00000100:00000010:0.0:1788574894.209773:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b505a0. 00000100:00000001:0.0:1788574894.209780:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574894.209786:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574894.209792:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.209797:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574894.209804:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574894/real 1788574894] req@ffff8fc4901a4a80 x1875440516754432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574894.209840:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574894.209842:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574894.209845:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574894.209853:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574894.209855:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574894.209861:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574894.209878:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516754432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574894.209909:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516754432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.209929:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574894.209933:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.209938:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574894.209943:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574894.209952:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516754432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.209969:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574894.209981:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516754432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210002:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574894.210010:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.210013:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574894.210015:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.210020:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574894.210024:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574894.210033:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574894.210043:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574894.210049:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574894.210060:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516754432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574894.210085:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516754432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574894.210097:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.210102:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574894.210110:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516754432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210166:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574894.210170:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574894.210175:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574894.210180:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574894.210182:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574894.210185:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574894.210188:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574894.210191:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788574894.210195:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574894.210197:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574894.210200:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788574894.210204:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574894.210206:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.210209:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.210217:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574894.210219:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.210222:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574894.210224:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574894.210241:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574894.210247:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516754560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210255:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574894.210257:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574894.210260:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574894.210262:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574894.210264:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574894.210267:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516754560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574894.210270:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574894.210273:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574894.210274:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574894.210276:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574894.210279:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574894.210281:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574894.210285:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574894.210292:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574894.210304:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574894.210308:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516754560, portal 10 00000100:00000001:0.0:1788574894.210314:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574894.210315:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788574894.210322:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516754560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210328:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574894.210330:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574894.210333:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574894.210336:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516754560, offset 0 00000400:00000200:0.0:1788574894.210340:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574894.210344:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574894.210348:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574894.210352:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574894.210354:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574894.210357:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574894.210359:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574894.210362:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547547:547547:256:4294967295] 192.168.202.128@tcp LPNI seq info [547470:547470:8:4294967295] 00000400:00000200:0.0:1788574894.210366:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574894.210373:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574894.210377:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574894.210381:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574894.210386:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574894.210389:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574894.210399:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574894.210401:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574894.210404:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574894.210408:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574894.210411:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574894.210413:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574894.210421:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574894.210425:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516754560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210433:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574894.210435:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516754560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210442:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.210443:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574894.210447:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574894.210450:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574894.210454:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574894.210456:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310254099 00000100:00000001:0.0:1788574894.210459:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310257901 00000100:00000001:0.0:1788574894.210461:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310254099 00000100:00000001:0.0:1788574894.210463:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310257901 00000100:00000001:0.0:1788574894.210465:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574894.210467:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.210469:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574894.210472:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574894/real 1788574894] req@ffff8fc4901a7100 x1875440516754560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574894.210480:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574894.210481:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574894.210483:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574894.210485:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574894.210487:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574894.210489:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574894.210493:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516754560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574894.210501:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516754560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210518:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574894.210520:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.210522:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574894.210523:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574894.210527:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516754560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210534:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574894.210538:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516754560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210545:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574894.210548:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.210549:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574894.210550:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.210552:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574894.210554:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574894.210556:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574894.210560:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574894.210562:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574894.210566:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516754560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574894.210574:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516754560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574894.210577:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574894.210581:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516754688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210587:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574894.210589:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574894.210595:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574894.210597:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574894.210599:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574894.210602:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516754688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574894.210606:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574894.210608:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574894.210609:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574894.210611:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574894.210614:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574894.210616:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574894.210619:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574894.210622:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574894.210626:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574894.210628:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516754688, portal 10 00000100:00000001:0.0:1788574894.210631:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574894.210632:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788574894.210638:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516754688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210645:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574894.210647:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574894.210649:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574894.210652:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516754688, offset 0 00000400:00000200:0.0:1788574894.210656:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574894.210659:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574894.210663:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574894.210666:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574894.210669:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574894.210671:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574894.210673:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574894.210676:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547548:547548:256:4294967295] 192.168.202.128@tcp LPNI seq info [547471:547471:8:4294967295] 00000400:00000200:0.0:1788574894.210680:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574894.210686:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574894.210695:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574894.210698:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574894.210702:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574894.210704:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574894.210715:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574894.210716:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574894.210719:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574894.210722:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574894.210725:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574894.210727:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574894.210729:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574894.210732:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516754688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210740:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574894.210743:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516754688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210749:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.210750:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574894.210753:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574894.210756:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574894.210759:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574894.210762:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310254106 00000100:00000001:0.0:1788574894.210764:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310257901 00000100:00000001:0.0:1788574894.210766:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310254106 00000100:00000001:0.0:1788574894.210768:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310257901 00000100:00000001:0.0:1788574894.210771:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574894.210773:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.210775:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574894.210779:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574894/real 1788574894] req@ffff8fc4901a5180 x1875440516754688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574894.210794:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574894.210796:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574894.210797:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574894.210800:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574894.210802:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574894.210804:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574894.210826:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516754688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574894.210836:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516754688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210843:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574894.210844:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.210846:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574894.210848:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574894.210851:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516754688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210858:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574894.210862:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516754688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210869:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574894.210872:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.210873:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574894.210875:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.210877:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574894.210879:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574894.210882:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574894.210886:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574894.210888:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574894.210899:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516754688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574894.210908:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516754688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574894.210912:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574894.210916:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516754816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210922:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574894.210924:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574894.210926:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574894.210928:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574894.210930:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574894.210933:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516754816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574894.210936:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574894.210938:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574894.210940:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574894.210941:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574894.210944:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574894.210946:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574894.210950:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574894.210952:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574894.210956:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574894.210958:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516754816, portal 4 00000100:00000001:0.0:1788574894.210960:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574894.210961:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788574894.210967:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516754816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.210973:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574894.210976:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574894.210978:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574894.210980:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516754816, offset 0 00000400:00000200:0.0:1788574894.210988:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574894.210992:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574894.210995:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574894.210998:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574894.211001:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574894.211003:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574894.211006:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574894.211009:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547549:547549:256:4294967295] 192.168.202.128@tcp LPNI seq info [547472:547472:8:4294967295] 00000400:00000200:0.0:1788574894.211013:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574894.211019:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574894.211023:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574894.211026:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574894.211030:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574894.211033:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574894.211044:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574894.211045:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574894.211048:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574894.211052:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574894.211055:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574894.211056:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574894.211059:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574894.211062:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516754816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211070:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574894.211074:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516754816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211080:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.211082:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574894.211085:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574894.211088:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574894.211091:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574894.211098:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310254247 00000100:00000001:0.0:1788574894.211100:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310257901 00000100:00000001:0.0:1788574894.211102:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310254247 00000100:00000001:0.0:1788574894.211104:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310257901 00000100:00000001:0.0:1788574894.211106:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574894.211108:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.211110:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574894.211114:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574894/real 1788574894] req@ffff8fc4901a5c00 x1875440516754816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574894.211123:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574894.211124:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574894.211126:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574894.211128:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574894.211130:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574894.211132:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574894.211136:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516754816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574894.211144:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516754816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211151:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574894.211153:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.211155:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574894.211156:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574894.211160:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516754816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211167:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574894.211171:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516754816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211177:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574894.211184:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.211185:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574894.211187:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.211188:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574894.211190:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574894.211193:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574894.211197:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574894.211199:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574894.211203:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516754816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574894.211213:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516754816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574894.211216:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574894.211220:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516754944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211242:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574894.211243:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574894.211246:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574894.211248:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574894.211250:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574894.211253:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516754944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574894.211257:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574894.211258:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574894.211260:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574894.211262:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574894.211266:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574894.211268:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574894.211271:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574894.211274:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574894.211277:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574894.211280:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516754944, portal 4 00000100:00000001:0.0:1788574894.211286:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574894.211288:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788574894.211294:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516754944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211300:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574894.211303:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574894.211305:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574894.211307:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516754944, offset 0 00000400:00000200:0.0:1788574894.211311:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574894.211315:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574894.211318:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574894.211321:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574894.211323:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574894.211325:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574894.211327:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574894.211331:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547550:547550:256:4294967295] 192.168.202.128@tcp LPNI seq info [547473:547473:8:4294967295] 00000400:00000200:0.0:1788574894.211335:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574894.211340:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574894.211344:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574894.211348:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574894.211351:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788574894.211354:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788574894.211363:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574894.211365:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56700. 00000400:00000200:0.0:1788574894.211368:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788574894.211371:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788574894.211374:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9000 00000400:00000010:0.0:1788574894.211375:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9000. 00000100:00000001:0.0:1788574894.211378:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788574894.211381:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516754944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211399:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574894.211402:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516754944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211409:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.211410:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788574894.211414:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574894.211417:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574894.211420:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574894.211422:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310254249 00000100:00000001:0.0:1788574894.211424:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310257901 00000100:00000001:0.0:1788574894.211426:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310254249 00000100:00000001:0.0:1788574894.211429:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310257901 00000100:00000001:0.0:1788574894.211431:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574894.211433:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.211435:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574894.211438:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574894/real 1788574894] req@ffff8fc4901a7800 x1875440516754944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574894.211446:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574894.211447:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574894.211449:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574894.211451:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574894.211453:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574894.211455:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574894.211458:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516754944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574894.211466:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516754944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211472:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574894.211479:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.211481:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574894.211482:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574894.211486:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516754944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211493:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574894.211497:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516754944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211504:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574894.211506:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.211507:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574894.211509:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.211510:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574894.211512:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574894.211515:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574894.211518:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574894.211520:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574894.211524:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516754944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574894.211533:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516754944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574894.211536:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.211538:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574894.211542:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516754560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211549:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574894.211550:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574894.211552:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574894.211555:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574894.211557:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574894.211562:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574894.211565:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574894.211567:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788574894.211570:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574894.211572:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574894.211573:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788574894.211578:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574894.211579:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.211581:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574894.211585:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516754688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211591:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574894.211593:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574894.211594:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574894.211597:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574894.211598:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574894.211599:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574894.211601:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574894.211602:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788574894.211604:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574894.211605:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574894.211607:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788574894.211611:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574894.211612:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.211614:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574894.211617:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516754816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211623:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574894.211624:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574894.211626:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574894.211628:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574894.211629:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574894.211630:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574894.211632:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574894.211633:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788574894.211635:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574894.211640:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574894.211642:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788574894.211645:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574894.211646:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.211647:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574894.211651:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516754944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574894.211658:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574894.211659:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574894.211661:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574894.211663:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574894.211665:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574894.211666:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574894.211668:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574894.211669:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788574894.211671:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574894.211673:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574894.211674:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788574894.211676:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574894.211677:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.211679:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.211683:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574894.211684:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.211686:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574894.211688:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.720907:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574894.720915:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574894.720917:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574894.720923:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574894.720930:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574894.720936:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.720944:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574894.720949:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.720954:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574894.720986:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574894.720989:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574894.720990:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574894.720992:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574894.721002:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574894.721004:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.721005:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574894.721007:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574894.721008:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574895.425024:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574895.425027:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574895.425031:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574895.744905:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574895.744911:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574895.744914:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574895.744919:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574895.744927:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574895.744932:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574895.744958:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574895.744961:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574895.744963:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574897.472974:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574897.472977:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574897.472979:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574897.792949:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574897.792955:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574897.792959:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574897.792965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574897.792970:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574897.792973:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574897.792982:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574897.792984:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574897.792988:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574898.048906:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310257901, at 4310257920 00000100:00000010:0.0:1788574898.048916:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574898.496974:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574898.496978:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574898.496983:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574898.816920:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574898.816928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574898.816932:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574898.816955:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574898.816962:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574898.816967:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574898.816982:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574898.816987:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574898.816994:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574899.329171:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574899.329178:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574899.329183:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574899.329226:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5c00 x1875440516755072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574899.329245:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574899.329249:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574899.329256:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574899.329261:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574899.329269:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574899.329278:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516755072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574899.329288:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574899.329295:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574899.329300:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574899.329307:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574899.329315:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574899.329322:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574899.329335:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574899.329343:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788574899.329356:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788574899.329366:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516755072, portal 25 00000100:00000001:0.0:1788574899.329373:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574899.329378:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605327872 : -123400104223744 : ffff8fc4ac9f5c00) 00000100:00000040:0.0:1788574899.329392:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5c00 x1875440516755072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788574924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574899.329410:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574899.329417:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574899.329424:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9000. 00000100:00000200:0.0:1788574899.329438:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516755072, offset 0 00000400:00000200:0.0:1788574899.329452:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574899.329468:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574899.329481:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574899.329491:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574899.329499:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574899.329506:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574899.329514:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574899.329521:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547551:547551:256:4294967295] 192.168.202.128@tcp LPNI seq info [547474:547474:8:4294967295] 00000400:00000200:0.0:1788574899.329532:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574899.329553:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574899.329566:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574899.329575:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56700. 00000800:00000200:0.0:1788574899.329586:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788574899.329620:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574899.329632:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574899.329638:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310263021 00000100:00000010:0.0:1788574899.329647:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788574899.329658:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788574899.329667:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310263021 00000100:00000010:0.0:1788574899.329676:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50f00. 00000100:00000001:0.0:1788574899.329684:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788574899.329692:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574899.329701:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574899.329709:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574899.329715:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574899.329727:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574899.329731:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788574899.329739:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574899.329743:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574899.329747:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574899.329756:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6300 x1875440516755200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574899.329776:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574899.329779:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574899.329783:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574899.329788:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574899.329791:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574899.329796:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516755200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574899.329799:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574899.329802:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574899.329804:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574899.329806:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574899.329828:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574899.329830:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574899.329834:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574899.329837:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19153c8. 00000400:00000010:0.0:1788574899.329841:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9c38. 00000100:00000200:0.0:1788574899.329845:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516755200, portal 10 00000100:00000001:0.0:1788574899.329849:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574899.329851:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605329664 : -123400104221952 : ffff8fc4ac9f6300) 00000100:00000040:0.0:1788574899.329865:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6300 x1875440516755200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574899.329887:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574899.329893:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574899.329898:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9d48. 00000100:00000200:0.0:1788574899.329903:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516755200, offset 0 00000400:00000200:0.0:1788574899.329912:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574899.329917:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574899.329922:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574899.329926:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574899.329929:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574899.329931:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574899.329933:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574899.329944:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547552:547552:255:4294967295] 192.168.202.128@tcp LPNI seq info [547475:547475:7:4294967295] 00000400:00000200:0.0:1788574899.329949:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574899.329956:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574899.329961:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574899.329965:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56f00. 00000800:00000200:0.0:1788574899.329970:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788574899.329977:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574899.329981:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788574899.329984:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574899.329985:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310259219 00000100:00000001:0.0:1788574899.329987:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310263021 00000100:00000001:0.0:1788574899.329989:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310259219 00000100:00000001:0.0:1788574899.329990:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310263021 00000100:00000001:0.0:1788574899.329992:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574899.329994:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574899.329995:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574899.329998:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4380 x1875440516755328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574899.330002:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574899.330004:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574899.330006:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574899.330008:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574899.330010:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574899.330013:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516755328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574899.330017:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574899.330019:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574899.330021:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574899.330022:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574899.330026:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574899.330028:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574899.330038:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574899.330041:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b00. 00000400:00000010:0.0:1788574899.330045:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788574899.330048:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516755328, portal 10 00000100:00000001:0.0:1788574899.330051:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574899.330052:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605321600 : -123400104230016 : ffff8fc4ac9f4380) 00000100:00000040:0.0:1788574899.330058:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4380 x1875440516755328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788574924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574899.330065:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574899.330067:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574899.330070:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed96e8. 00000100:00000200:0.0:1788574899.330072:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516755328, offset 0 00000400:00000200:0.0:1788574899.330077:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574899.330081:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574899.330085:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574899.330088:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574899.330091:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574899.330094:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574899.330097:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574899.330100:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547553:547553:254:4294967295] 192.168.202.128@tcp LPNI seq info [547476:547476:6:4294967295] 00000400:00000200:0.0:1788574899.330105:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574899.330112:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574899.330118:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574899.330121:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56100. 00000800:00000200:0.0:1788574899.330126:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788574899.330132:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574899.330135:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788574899.330139:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574899.330141:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310259226 00000100:00000001:0.0:1788574899.330143:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310263021 00000100:00000001:0.0:1788574899.330145:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310259226 00000100:00000001:0.0:1788574899.330153:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310263021 00000100:00000001:0.0:1788574899.330156:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574899.330159:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574899.330161:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574899.330165:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5500 x1875440516755456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574899.330173:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574899.330174:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574899.330177:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574899.330179:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574899.330181:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574899.330184:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516755456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574899.330202:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574899.330204:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574899.330205:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574899.330207:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574899.330210:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574899.330212:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574899.330216:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574899.330219:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915e70. 00000400:00000010:0.0:1788574899.330224:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788574899.330227:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516755456, portal 4 00000100:00000001:0.0:1788574899.330229:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574899.330230:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605326080 : -123400104225536 : ffff8fc4ac9f5500) 00000100:00000040:0.0:1788574899.330236:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5500 x1875440516755456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574899.330243:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574899.330246:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574899.330249:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9198. 00000100:00000200:0.0:1788574899.330251:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516755456, offset 0 00000400:00000200:0.0:1788574899.330256:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574899.330260:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574899.330270:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574899.330274:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574899.330276:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574899.330278:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574899.330280:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574899.330283:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547554:547554:253:4294967295] 192.168.202.128@tcp LPNI seq info [547477:547477:5:4294967295] 00000400:00000200:0.0:1788574899.330287:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574899.330294:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574899.330299:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574899.330303:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56800. 00000800:00000200:0.0:1788574899.330307:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788574899.330314:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574899.330317:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788574899.330320:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574899.330323:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310259366 00000100:00000001:0.0:1788574899.330325:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310263021 00000100:00000001:0.0:1788574899.330327:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310259366 00000100:00000001:0.0:1788574899.330329:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310263021 00000100:00000001:0.0:1788574899.330332:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574899.330334:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574899.330336:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574899.330338:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574899.330343:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574899.330345:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788574899.330347:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574899.330348:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574899.330349:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788574899.330353:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4e00 x1875440516755584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574899.330360:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574899.330367:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574899.330370:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788574899.330372:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574899.330374:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574899.330377:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516755584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574899.330381:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788574899.330383:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788574899.330385:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574899.330387:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788574899.330391:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574899.330393:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788574899.330397:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788574899.330399:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788574899.330403:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9b28. 00000100:00000200:0.0:1788574899.330405:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516755584, portal 4 00000100:00000001:0.0:1788574899.330407:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788574899.330409:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605324288 : -123400104227328 : ffff8fc4ac9f4e00) 00000100:00000040:0.0:1788574899.330415:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4e00 x1875440516755584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788574924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574899.330421:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788574899.330423:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788574899.330426:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9990. 00000100:00000200:0.0:1788574899.330428:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516755584, offset 0 00000400:00000200:0.0:1788574899.330433:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788574899.330437:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574899.330441:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788574899.330444:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788574899.330446:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788574899.330449:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788574899.330451:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788574899.330454:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547555:547555:252:4294967295] 192.168.202.128@tcp LPNI seq info [547478:547478:4:4294967295] 00000400:00000200:0.0:1788574899.330458:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574899.330465:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788574899.330476:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788574899.330480:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56400. 00000800:00000200:0.0:1788574899.330484:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788574899.330491:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574899.330494:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788574899.330498:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788574899.330500:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310259368 00000100:00000001:0.0:1788574899.330503:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310263021 00000100:00000001:0.0:1788574899.330505:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310259368 00000100:00000001:0.0:1788574899.330507:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310263021 00000100:00000001:0.0:1788574899.330510:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788574899.330512:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574899.330515:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574899.330517:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574899.330521:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574899.330523:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788574899.330525:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574899.330526:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574899.330528:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574899.330530:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574899.840932:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574899.840939:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574899.840944:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574899.840952:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574899.840958:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574899.840964:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574899.840977:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574899.840980:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574899.840983:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574900.545072:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574900.545076:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574900.545078:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574900.864939:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574900.864956:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574900.864961:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574900.864969:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574900.864978:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574900.864985:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574900.864998:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574900.865002:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574900.865008:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574901.888902:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574901.888911:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574901.888916:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574901.888926:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574901.888934:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574901.888941:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574901.888949:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574901.888954:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574901.888960:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574901.889009:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574901.889013:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574901.889015:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574901.889019:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574901.889022:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574901.889024:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574901.889027:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574901.889028:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574901.889030:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574902.401575:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574902.401580:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574902.401583:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574902.401591:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574899/real 1788574902] req@ffff8fc4ac9f4e00 x1875440516755584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574902.401605:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574902.401607:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574902.401611:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788574902.401615:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9b28 00000400:00000010:0.0:1788574902.401617:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9b28. 00000100:00000001:0.0:1788574902.401620:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574902.401635:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4e00 x1875440516755584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574902.401654:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4e00 x1875440516755584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.401661:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574902.401663:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.401666:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574902.401669:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574902.401675:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4e00 x1875440516755584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.401683:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574902.401688:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4e00 x1875440516755584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.401697:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574902.401700:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.401702:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574902.401703:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.401706:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574902.401709:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574902.401712:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574902.401717:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574902.401720:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574902.401724:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4e00 x1875440516755584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574902.401734:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516755584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574902.401738:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574902.401742:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574899/real 1788574902] req@ffff8fc4ac9f6300 x1875440516755200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574902.401764:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574902.401765:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574902.401768:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19153c8. 00000400:00000200:0.0:1788574902.401771:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9c38 00000400:00000010:0.0:1788574902.401772:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9c38. 00000100:00000001:0.0:1788574902.401775:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574902.401779:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6300 x1875440516755200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574902.401787:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6300 x1875440516755200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.401793:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574902.401794:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.401796:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574902.401797:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574902.401801:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6300 x1875440516755200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.401829:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574902.401834:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6300 x1875440516755200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.401842:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574902.401844:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.401846:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574902.401848:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.401850:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574902.401852:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574902.401855:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574902.401858:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574902.401860:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574902.401872:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6300 x1875440516755200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574902.401880:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516755200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574902.401885:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574902.401889:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574899/real 1788574902] req@ffff8fc4ac9f4380 x1875440516755328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574902.401897:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574902.401899:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574902.401901:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b00. 00000400:00000200:0.0:1788574902.401904:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788574902.401906:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788574902.401909:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574902.401913:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4380 x1875440516755328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574902.401921:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4380 x1875440516755328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.401928:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574902.401930:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.401931:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574902.401933:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574902.401937:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4380 x1875440516755328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.401945:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574902.401949:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4380 x1875440516755328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.401971:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574902.401973:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.401982:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574902.401984:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.401986:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574902.401989:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574902.401991:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574902.401996:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574902.401999:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574902.402003:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4380 x1875440516755328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574902.402012:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516755328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788574902.402017:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574902.402021:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574899/real 1788574902] req@ffff8fc4ac9f5500 x1875440516755456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574902.402030:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574902.402032:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574902.402034:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915e70. 00000400:00000200:0.0:1788574902.402036:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788574902.402038:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788574902.402041:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574902.402044:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5500 x1875440516755456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574902.402052:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5500 x1875440516755456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.402057:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574902.402059:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402061:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574902.402062:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574902.402065:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5500 x1875440516755456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.402082:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574902.402087:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5500 x1875440516755456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.402093:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574902.402096:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402098:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574902.402099:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402101:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574902.402104:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574902.402106:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574902.402110:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574902.402112:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574902.402116:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5500 x1875440516755456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574902.402123:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516755456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788574902.402127:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788574902.402131:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788574899/real 1788574902] req@ffff8fc4ac9f5c00 x1875440516755072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788574902.402140:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788574902.402142:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788574902.402144:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788574902.402146:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788574902.402149:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788574902.402152:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788574902.402154:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5c00 x1875440516755072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788574902.402175:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5c00 x1875440516755072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.402188:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788574902.402190:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402192:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574902.402193:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788574902.402197:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5c00 x1875440516755072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.402204:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788574902.402208:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5c00 x1875440516755072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.402216:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788574902.402219:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402222:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788574902.402223:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402225:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788574902.402227:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788574902.402230:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788574902.402235:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788574902.402238:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788574902.402243:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5c00 x1875440516755072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788574902.402253:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516755072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788574902.402258:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402261:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574902.402264:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4e00 x1875440516755584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.402272:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574902.402275:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574902.402284:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788574902.402288:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574902.402291:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574902.402294:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574902.402296:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574902.402299:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37c00. 02000000:00000001:0.0:1788574902.402303:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574902.402305:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574902.402307:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4e00. 00000100:00000001:0.0:1788574902.402311:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574902.402313:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402315:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574902.402319:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6300 x1875440516755200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.402327:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574902.402328:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574902.402330:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788574902.402332:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574902.402334:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574902.402335:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574902.402338:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574902.402339:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788574902.402342:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574902.402343:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574902.402345:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6300. 00000100:00000001:0.0:1788574902.402348:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574902.402349:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402361:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574902.402368:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4380 x1875440516755328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.402374:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574902.402375:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574902.402377:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788574902.402380:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574902.402381:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574902.402382:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788574902.402394:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574902.402396:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788574902.402399:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574902.402400:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574902.402401:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4380. 00000100:00000001:0.0:1788574902.402404:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574902.402405:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402407:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574902.402411:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5500 x1875440516755456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.402419:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574902.402420:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574902.402422:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788574902.402425:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574902.402427:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574902.402428:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788574902.402430:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574902.402432:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788574902.402434:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574902.402435:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574902.402436:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5500. 00000100:00000001:0.0:1788574902.402438:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574902.402440:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402441:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788574902.402445:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5c00 x1875440516755072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788574924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574902.402450:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788574902.402452:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788574902.402454:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788574902.402456:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788574902.402457:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788574902.402458:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788574902.402460:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788574902.402461:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35800. 02000000:00000001:0.0:1788574902.402464:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788574902.402465:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788574902.402472:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5c00. 00000100:00000001:0.0:1788574902.402475:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788574902.402476:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402478:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.402485:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574902.402488:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574902.402490:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788574902.402491:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574902.593046:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574902.593050:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574902.593053:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574902.912919:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574902.912926:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574902.912931:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574902.912937:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574902.912944:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574902.912949:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.912963:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574902.912967:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574902.912972:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574903.168922:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310263021, at 4310263040 00000100:00000010:0.0:1788574903.168938:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788574903.617017:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574903.617020:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574903.617023:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574903.936910:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574903.936918:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574903.936922:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574903.936928:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574903.936935:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574903.936940:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574903.936957:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574903.936962:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574903.936967:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574903.937003:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574903.937006:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574903.937008:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574903.937017:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574903.937020:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574903.937022:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574903.937024:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574903.937026:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574903.937028:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574904.448953:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574904.448964:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574904.448967:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574904.448971:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574904.448976:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574904.448984:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15591 00000400:00000200:0.0:1788574904.448993:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574904.448999:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574904.449003:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574904.449006:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449009:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574904.449011:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449012:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574904.449016:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574904.449017:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574904.449019:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574904.449021:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574904.449023:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574904.449025:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574904.449027:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449028:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574904.449029:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449031:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574904.449032:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574904.449035:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449051:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574904.449054:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574904.449057:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574904.449060:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449063:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574904.449065:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574904.449070:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574904.449074:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574904.449078:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574904.449080:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449083:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574904.449086:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574904.449090:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574904.449098:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574904.449105:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574904.449114:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516755712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574904.449123:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574904.449127:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574904.449132:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516755712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574904.449186:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574904.449190:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574904.449194:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574904.449195:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574904.449197:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574904.449200:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574904.449203:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15591 00000400:00000200:0.0:1788574904.449208:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574904.449213:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574904.449215:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574904.449219:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449231:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574904.449236:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449240:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574904.449243:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574904.449244:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574904.449247:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574904.449252:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574904.449256:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574904.449260:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574904.449269:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449274:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574904.449280:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449286:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574904.449291:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574904.449297:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449303:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574904.449309:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574904.449318:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574904.449323:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449328:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574904.449334:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574904.449340:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449344:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574904.449351:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574904.449356:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574904.449360:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574904.449362:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449367:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574904.449369:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574904.449372:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574904.449379:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574904.449391:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574904.449409:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516755840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574904.449433:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574904.449438:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574904.449446:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516755840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574904.449461:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574904.449466:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574904.449474:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574904.449477:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574904.449479:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574904.449484:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574904.449491:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15591 00000400:00000200:0.0:1788574904.449501:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574904.449509:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574904.449515:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574904.449520:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449524:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574904.449531:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449536:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574904.449542:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574904.449545:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574904.449549:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574904.449554:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574904.449559:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574904.449564:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574904.449572:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449576:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574904.449583:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449597:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574904.449603:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574904.449609:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449614:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574904.449619:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574904.449625:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574904.449632:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449636:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574904.449639:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574904.449642:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449644:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574904.449650:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574904.449655:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574904.449661:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574904.449665:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449669:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574904.449671:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574904.449676:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574904.449681:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574904.449685:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574904.449691:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516755968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574904.449711:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574904.449715:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574904.449719:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516755968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574904.449726:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574904.449730:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574904.449733:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574904.449735:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574904.449736:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574904.449753:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574904.449757:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15591 00000400:00000200:0.0:1788574904.449762:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574904.449765:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574904.449767:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574904.449770:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449772:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574904.449774:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449776:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574904.449778:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574904.449780:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574904.449781:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574904.449783:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574904.449785:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574904.449787:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574904.449790:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449792:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574904.449794:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449796:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574904.449798:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574904.449801:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449803:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574904.449805:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574904.449833:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574904.449836:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449837:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574904.449840:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574904.449842:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449844:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574904.449848:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574904.449856:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574904.449858:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574904.449859:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449861:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574904.449863:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574904.449865:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574904.449868:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574904.449872:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574904.449877:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516756096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574904.449885:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574904.449887:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574904.449891:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516756096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574904.449898:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574904.449901:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574904.449905:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574904.449906:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574904.449908:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574904.449910:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574904.449918:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15591 00000400:00000200:0.0:1788574904.449922:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574904.449926:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574904.449928:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574904.449931:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449933:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574904.449935:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449937:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574904.449939:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574904.449940:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574904.449948:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574904.449950:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574904.449953:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574904.449955:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574904.449958:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449960:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574904.449962:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449964:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574904.449966:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574904.449969:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574904.449973:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574904.449975:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574904.449978:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574904.449980:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449982:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574904.449984:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574904.449987:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.449989:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574904.449992:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574904.449995:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574904.449998:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574904.449999:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574904.450001:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574904.450003:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574904.450005:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574904.450011:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574904.450015:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574904.450021:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516756224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574904.450030:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574904.450032:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574904.450037:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516756224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574904.450055:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574904.450062:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15602) 00000100:00000001:0.0:1788574904.960915:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574904.960923:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574904.960927:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574904.960935:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574904.960941:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574904.960947:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574904.960961:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574904.960966:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574904.960972:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574905.665074:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574905.665077:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574905.665079:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574905.984905:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574905.984914:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574905.984918:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574905.984926:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574905.984933:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574905.984939:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574905.984944:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574905.984950:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574905.984954:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574905.984972:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574905.984978:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574905.984981:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574905.984986:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574905.984989:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574905.984993:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574905.984995:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574905.984997:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574905.984999:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574907.008935:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574907.008941:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574907.008947:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574907.008952:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574907.008976:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574907.008981:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574907.008986:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574907.008993:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574907.009002:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574907.713044:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574907.713048:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574907.713052:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574908.032926:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574908.032933:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574908.032936:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574908.032944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574908.032949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574908.032954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574908.032960:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574908.032965:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574908.032970:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574908.033005:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574908.033009:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574908.033011:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574908.033014:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574908.033017:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574908.033020:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574908.033021:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574908.033022:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574908.033024:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574908.288910:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310268142, at 4310268160 00000100:00000010:0.0:1788574908.288933:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf900. 00000020:00000001:0.0:1788574908.736979:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574908.736984:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574908.736988:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574909.056966:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574909.056971:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574909.056973:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574909.056977:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574909.056981:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574909.056983:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574909.057002:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574909.057006:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574909.057010:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574910.080921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574910.080929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574910.080933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574910.080937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574910.080942:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574910.080946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574910.080950:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574910.080952:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574910.080959:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574910.785128:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574910.785132:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574910.785135:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574911.104994:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574911.104998:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574911.105000:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574911.105002:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574911.105011:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574911.105014:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574911.105018:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574911.105021:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574911.105025:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574912.128926:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574912.128934:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574912.128938:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574912.128956:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574912.128961:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574912.128966:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574912.128986:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574912.128989:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574912.128991:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574912.833028:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574912.833032:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574912.833034:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574913.152927:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574913.152934:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574913.152953:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574913.152957:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574913.152961:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574913.152965:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574913.152990:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574913.152993:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574913.152998:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574913.408899:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310273261, at 4310273280 00000100:00000010:0.0:1788574913.408910:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfcc0. 00000020:00000001:0.0:1788574913.856979:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574913.856982:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574913.856984:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574915.200930:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574915.200935:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574915.200938:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574915.200942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574915.200944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574915.200946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574915.200948:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574915.200949:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574915.200952:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574915.201077:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574915.201082:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574915.201085:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574915.201092:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574915.201095:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574915.201099:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574915.201103:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574915.201105:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574915.201110:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574915.904993:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574915.905003:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574915.905005:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574916.224933:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574916.224940:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574916.224944:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574916.224952:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574916.224977:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574916.224983:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574916.224990:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574916.224995:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574916.224999:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574916.737088:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788574917.248938:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574917.248945:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574917.248948:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574917.248954:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574917.248959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574917.248964:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574917.248968:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574917.248974:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574917.248981:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574917.952972:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574917.952976:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574917.952979:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574918.272928:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574918.272935:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574918.272939:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574918.272949:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574918.272959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574918.272964:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574918.272967:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574918.272972:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574918.272976:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574918.528914:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310278381, at 4310278400 00000100:00000010:0.0:1788574918.528932:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfa20. 00000020:00000001:0.0:1788574918.976991:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574918.976994:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574918.976996:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574919.296909:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574919.296914:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574919.296917:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574919.296923:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574919.296930:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574919.296951:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574919.296955:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574919.296959:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574919.296963:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574919.808922:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574919.808931:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574919.808933:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574919.808936:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574919.808940:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574919.808958:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15607 00000400:00000200:0.0:1788574919.808966:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574919.808973:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574919.808975:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574919.808979:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.808982:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574919.808985:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.808987:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574919.808990:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574919.808992:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574919.808994:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574919.808997:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574919.808999:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574919.809001:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574919.809003:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809004:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574919.809005:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809007:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574919.809009:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574919.809011:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809012:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574919.809014:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574919.809044:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574919.809047:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809048:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574919.809050:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574919.809055:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574919.809058:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574919.809061:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574919.809062:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809065:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574919.809067:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574919.809071:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574919.809076:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574919.809082:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574919.809089:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516757632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574919.809097:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574919.809101:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574919.809104:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516757632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574919.809120:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574919.809123:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574919.809125:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574919.809126:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574919.809127:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574919.809130:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574919.809132:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15607 00000400:00000200:0.0:1788574919.809136:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574919.809139:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574919.809140:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574919.809144:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809147:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574919.809155:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809158:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574919.809162:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574919.809163:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574919.809166:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574919.809168:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574919.809170:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574919.809174:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574919.809181:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809186:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574919.809189:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809194:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574919.809197:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574919.809203:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809209:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574919.809211:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574919.809218:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574919.809223:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809225:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574919.809230:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574919.809234:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809238:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574919.809243:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574919.809246:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574919.809250:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574919.809251:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809252:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574919.809257:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574919.809259:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574919.809266:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574919.809271:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574919.809277:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516757760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574919.809296:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574919.809300:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574919.809305:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516757760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574919.809316:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574919.809322:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574919.809328:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574919.809330:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574919.809333:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574919.809339:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574919.809352:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15607 00000400:00000200:0.0:1788574919.809359:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574919.809363:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574919.809365:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574919.809371:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809373:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574919.809378:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809381:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574919.809386:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574919.809387:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574919.809388:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574919.809389:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574919.809391:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574919.809394:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574919.809400:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809402:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574919.809406:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809412:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574919.809420:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574919.809427:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809433:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574919.809434:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574919.809438:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574919.809441:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809442:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574919.809444:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574919.809447:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809450:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574919.809455:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574919.809460:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574919.809464:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574919.809467:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809472:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574919.809476:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574919.809477:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574919.809482:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574919.809488:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574919.809495:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516757888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574919.809502:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574919.809505:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574919.809511:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516757888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574919.809526:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574919.809531:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574919.809536:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574919.809540:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574919.809543:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574919.809549:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574919.809561:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15607 00000400:00000200:0.0:1788574919.809569:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574919.809572:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574919.809574:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574919.809576:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809578:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574919.809583:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809587:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574919.809590:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574919.809593:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574919.809596:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574919.809598:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574919.809603:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574919.809607:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574919.809614:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809617:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574919.809620:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809623:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574919.809627:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574919.809634:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809639:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574919.809643:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574919.809646:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574919.809651:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809653:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574919.809655:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574919.809659:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809663:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574919.809669:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574919.809672:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574919.809673:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574919.809679:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809682:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574919.809684:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574919.809686:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574919.809692:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574919.809695:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574919.809703:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516758016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574919.809714:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574919.809716:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574919.809719:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516758016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574919.809729:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574919.809733:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574919.809739:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574919.809742:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574919.809744:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574919.809749:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574919.809754:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15607 00000400:00000200:0.0:1788574919.809761:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574919.809765:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574919.809769:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574919.809787:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809795:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574919.809802:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809842:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574919.809848:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574919.809851:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574919.809854:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574919.809858:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574919.809863:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574919.809873:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574919.809880:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809884:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574919.809888:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809890:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574919.809892:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574919.809897:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574919.809901:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574919.809906:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574919.809911:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574919.809916:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809921:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574919.809924:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574919.809929:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809931:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574919.809938:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574919.809944:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574919.809950:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574919.809953:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574919.809958:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574919.809961:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574919.809964:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574919.809970:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574919.809975:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574919.809984:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516758144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574919.809997:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574919.810000:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574919.810007:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516758144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574919.810023:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574919.810047:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15617) 00000100:00000001:0.0:1788574920.320943:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574920.320952:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574920.320958:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574920.320967:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574920.320974:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574920.320979:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574920.320998:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574920.321002:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574920.321005:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574921.025062:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574921.025066:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574921.025069:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574921.345100:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574921.345103:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574921.345104:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574921.345106:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574921.345108:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574921.345110:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574921.345119:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574921.345120:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574921.345122:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574922.368981:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574922.368994:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574922.369000:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574922.369027:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574922.369036:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574922.369044:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574922.369065:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574922.369071:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574922.369076:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574923.073033:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574923.073040:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574923.073046:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574923.392917:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574923.392928:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574923.392934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574923.392945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574923.392981:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574923.392989:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574923.393018:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574923.393026:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574923.393032:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574923.648978:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310283501, at 4310283520 00000100:00000010:0.0:1788574923.648994:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf720. 00000020:00000001:0.0:1788574924.096959:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574924.096962:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574924.096965:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574924.417053:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574924.417057:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574924.417058:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574924.417061:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574924.417062:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574924.417064:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574924.417065:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574924.417067:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574924.417068:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.417076:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574924.417077:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574924.417078:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574924.417079:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574924.417081:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574924.417082:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574924.417083:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574924.417084:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574924.417085:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574924.928970:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574924.928986:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574924.929003:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574924.929009:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574924.929014:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574924.929025:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15612 00000400:00000200:0.0:1788574924.929038:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574924.929071:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574924.929076:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574924.929080:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929084:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574924.929087:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929090:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574924.929095:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574924.929098:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574924.929102:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574924.929105:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574924.929108:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574924.929111:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574924.929114:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929117:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574924.929119:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929121:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574924.929125:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574924.929130:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929133:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574924.929136:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574924.929141:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574924.929145:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.929148:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574924.929151:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574924.929161:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788574924.929166:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574924.929171:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574924.929173:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.929177:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574924.929180:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574924.929186:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574924.929198:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574924.929214:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574924.929227:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516758272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574924.929240:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574924.929246:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574924.929252:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516758272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574924.929276:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574924.929280:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574924.929284:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574924.929287:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574924.929289:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574924.929292:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574924.929297:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15612 00000400:00000200:0.0:1788574924.929304:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574924.929312:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574924.929318:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574924.929327:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929333:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574924.929340:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929346:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574924.929353:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574924.929358:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574924.929362:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574924.929368:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574924.929379:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574924.929383:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574924.929390:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929396:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574924.929412:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929417:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574924.929421:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574924.929428:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929433:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574924.929439:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574924.929449:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574924.929453:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.929456:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574924.929464:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574924.929471:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.929475:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574924.929483:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788574924.929487:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574924.929491:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574924.929493:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.929496:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574924.929501:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574924.929505:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574924.929515:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574924.929523:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574924.929536:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516758400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574924.929559:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574924.929562:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574924.929574:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516758400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574924.929594:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574924.929600:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574924.929607:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574924.929610:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574924.929620:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574924.929624:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574924.929630:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15612 00000400:00000200:0.0:1788574924.929639:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574924.929647:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574924.929652:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574924.929659:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929663:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574924.929668:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929674:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574924.929679:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574924.929681:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574924.929687:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574924.929690:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574924.929695:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574924.929698:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574924.929707:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929714:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574924.929719:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929724:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574924.929729:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574924.929736:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574924.929742:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574924.929748:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574924.929754:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574924.929762:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.929765:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574924.929769:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574924.929776:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.929789:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574924.929802:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788574924.929833:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574924.929838:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574924.929840:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.929843:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574924.929845:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574924.929848:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574924.929858:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574924.929865:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574924.929875:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516758528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574924.929889:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574924.929895:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574924.929906:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516758528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574924.929924:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574924.929930:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574924.929938:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574924.929941:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574924.929943:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574924.929950:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574924.929956:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15612 00000400:00000200:0.0:1788574924.929967:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574924.929976:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574924.929982:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574924.930003:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.930010:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574924.930019:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.930026:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574924.930032:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574924.930044:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574924.930050:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574924.930056:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574924.930062:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574924.930069:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574924.930079:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.930085:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574924.930091:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.930097:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574924.930104:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574924.930114:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574924.930121:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574924.930126:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574924.930133:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574924.930141:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.930145:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574924.930149:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574924.930157:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.930162:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574924.930168:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788574924.930173:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574924.930178:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574924.930183:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.930186:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574924.930190:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574924.930195:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574924.930202:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574924.930210:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574924.930221:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516758656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574924.930237:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574924.930243:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574924.930262:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516758656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574924.930289:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574924.930296:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574924.930306:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574924.930311:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574924.930314:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574924.930321:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574924.930328:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15612 00000400:00000200:0.0:1788574924.930340:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574924.930348:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574924.930355:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574924.930365:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.930373:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574924.930380:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.930388:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574924.930395:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574924.930400:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574924.930406:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574924.930412:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574924.930419:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574924.930425:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574924.930435:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.930442:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574924.930449:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574924.930457:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574924.930466:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574924.930474:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574924.930483:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574924.930494:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574924.930499:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574924.930505:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.930512:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574924.930517:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574924.930524:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.930530:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574924.930539:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788574924.930543:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574924.930548:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574924.930553:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574924.930558:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574924.930563:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574924.930568:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574924.930578:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574924.930585:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574924.930596:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516758784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574924.930617:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574924.930621:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574924.930631:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516758784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574924.930650:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574924.930655:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15622) 00000020:00000001:0.0:1788574926.144997:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574926.144999:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574926.145002:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574926.464927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574926.464936:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574926.464941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574926.464949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574926.464957:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574926.464964:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574926.464989:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574926.465010:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574926.465016:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574927.488877:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574927.488885:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574927.488888:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574927.488896:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574927.488904:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574927.488910:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574927.488916:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574927.488918:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574927.488932:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574927.488963:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574927.488981:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574927.488985:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574927.488991:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574927.488994:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574927.488996:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574927.488998:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574927.488999:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574927.489001:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574928.193020:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574928.193023:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574928.193026:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574928.512919:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574928.512925:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574928.512930:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574928.512937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574928.512944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574928.512948:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574928.512981:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574928.512985:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574928.512988:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574928.768922:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310288621, at 4310288640 00000100:00000010:0.0:1788574928.768937:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cff60. 00000020:00000001:0.0:1788574929.217095:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574929.217099:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574929.217103:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574929.537021:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574929.537035:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574929.537048:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574929.537057:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574929.537065:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574929.537072:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574929.537116:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574929.537126:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574929.537137:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574930.560978:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574930.560990:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574930.560996:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574930.561007:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574930.561017:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574930.561025:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574930.561035:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574930.561040:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574930.561048:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574931.265127:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574931.265132:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574931.265136:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574931.585025:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574931.585036:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574931.585043:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574931.585059:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574931.585077:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574931.585087:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574931.585105:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574931.585112:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574931.585121:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574932.608906:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574932.608913:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574932.608916:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574932.608921:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574932.608926:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574932.608941:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574932.608946:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574932.608950:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574932.608972:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574933.313043:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574933.313046:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574933.313049:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788574933.888932:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310293742, at 4310293760 00000100:00000010:0.0:1788574933.888948:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf420. 00000020:00000001:0.0:1788574934.337052:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574934.337056:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574934.337060:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574934.656964:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574934.656970:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574934.656975:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574934.656980:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574934.656985:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574934.656990:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574934.657011:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574934.657015:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574934.657018:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574935.680960:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574935.680965:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574935.680969:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574935.680972:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574935.680974:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574935.680979:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574935.680994:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574935.680997:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574935.681001:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574936.385058:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574936.385063:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574936.385066:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574936.705012:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574936.705020:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574936.705023:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574936.705032:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574936.705037:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574936.705040:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574936.705043:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574936.705060:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574936.705064:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574937.728938:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574937.728948:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574937.728953:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574937.728963:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574937.728970:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574937.728975:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574937.728992:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574937.728996:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574937.728999:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574937.729029:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574937.729034:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574937.729035:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574937.729039:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574937.729041:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574937.729044:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574937.729046:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574937.729048:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574937.729050:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574938.433076:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574938.433081:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574938.433084:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574938.752924:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574938.752931:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574938.752935:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574938.752942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574938.752947:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574938.752950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574938.752954:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574938.752957:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574938.752962:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574939.008911:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310298861, at 4310298880 00000100:00000010:0.0:1788574939.008927:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf540. 00000020:00000001:0.0:1788574939.456995:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574939.456998:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574939.457000:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574939.776928:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574939.776939:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574939.776941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574939.776949:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574939.776959:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574939.776968:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574939.776975:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574939.776984:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574939.776997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574940.288987:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574940.289002:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574940.289016:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574940.289021:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574940.289027:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574940.289036:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15627 00000400:00000200:0.0:1788574940.289049:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574940.289057:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574940.289061:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574940.289064:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289068:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574940.289071:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289074:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574940.289079:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574940.289081:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574940.289084:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574940.289087:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574940.289089:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574940.289092:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574940.289094:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289096:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574940.289098:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289101:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574940.289127:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574940.289132:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289135:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574940.289138:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574940.289142:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574940.289145:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.289148:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574940.289150:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574940.289158:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788574940.289162:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574940.289167:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574940.289169:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.289172:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574940.289176:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574940.289181:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574940.289192:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788574940.289200:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574940.289211:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516760192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574940.289223:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574940.289228:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574940.289234:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516760192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574940.289257:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574940.289261:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574940.289264:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574940.289266:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574940.289268:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574940.289271:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574940.289274:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15627 00000400:00000200:0.0:1788574940.289280:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574940.289291:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574940.289295:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574940.289301:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289306:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574940.289311:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289316:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574940.289322:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574940.289325:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574940.289330:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574940.289335:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574940.289340:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574940.289343:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574940.289350:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289357:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574940.289364:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289370:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574940.289375:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574940.289380:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289386:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574940.289389:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574940.289399:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574940.289404:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.289408:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574940.289414:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574940.289422:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.289427:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574940.289435:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788574940.289442:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574940.289448:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574940.289452:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.289458:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574940.289470:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574940.289474:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574940.289481:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574940.289487:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574940.289496:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516760320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574940.289514:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574940.289520:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574940.289529:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516760320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574940.289552:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574940.289558:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574940.289564:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574940.289566:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574940.289569:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574940.289575:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574940.289590:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15627 00000400:00000200:0.0:1788574940.289599:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574940.289606:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574940.289609:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574940.289614:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289616:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574940.289618:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289625:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574940.289629:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574940.289632:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574940.289636:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574940.289639:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574940.289642:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574940.289646:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574940.289660:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289665:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574940.289671:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289675:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574940.289679:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574940.289684:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289689:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574940.289693:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574940.289697:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574940.289702:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.289706:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574940.289711:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574940.289715:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.289717:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574940.289721:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788574940.289726:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574940.289731:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574940.289733:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.289737:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574940.289739:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574940.289742:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574940.289748:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574940.289753:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574940.289760:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516760448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574940.289770:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574940.289773:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574940.289780:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516760448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574940.289795:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574940.289801:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574940.289870:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574940.289885:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574940.289889:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574940.289894:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574940.289901:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15627 00000400:00000200:0.0:1788574940.289910:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574940.289916:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574940.289920:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574940.289928:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289932:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574940.289937:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289943:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574940.289947:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574940.289949:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574940.289952:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574940.289955:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574940.289960:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574940.289963:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574940.289971:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289973:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574940.289978:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289980:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574940.289984:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574940.289990:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574940.289994:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574940.289998:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574940.290003:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574940.290007:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.290011:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574940.290013:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574940.290024:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.290028:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574940.290034:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788574940.290039:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574940.290045:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574940.290048:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.290051:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574940.290054:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574940.290058:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574940.290063:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574940.290069:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574940.290078:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516760576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574940.290088:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574940.290093:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574940.290101:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516760576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574940.290118:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574940.290123:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574940.290130:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574940.290134:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574940.290138:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574940.290143:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574940.290149:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15627 00000400:00000200:0.0:1788574940.290156:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574940.290162:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574940.290167:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574940.290174:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.290178:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574940.290184:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.290195:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574940.290201:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574940.290205:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574940.290208:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574940.290212:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574940.290217:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574940.290224:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574940.290232:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.290237:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574940.290243:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574940.290249:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574940.290255:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574940.290262:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574940.290268:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574940.290273:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574940.290279:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574940.290286:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.290291:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574940.290296:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574940.290301:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.290305:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574940.290312:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788574940.290317:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574940.290322:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574940.290325:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.290328:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574940.290331:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574940.290335:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574940.290340:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574940.290346:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574940.290356:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516760704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574940.290392:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574940.290396:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574940.290404:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516760704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574940.290425:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574940.290430:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15637) 00000100:00000001:0.0:1788574940.800956:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574940.800966:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574940.800972:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574940.800980:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574940.800986:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574940.800990:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574940.800995:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574940.800999:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574940.801002:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574940.801100:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574940.801105:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574940.801109:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574940.801117:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574940.801123:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574940.801125:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574940.801128:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574940.801130:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574940.801132:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574941.505022:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574941.505028:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574941.505031:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574941.825192:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574941.825196:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574941.825198:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574941.825202:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574941.825205:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574941.825207:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574941.825333:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574941.825357:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574941.825361:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574942.848925:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574942.848929:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574942.848932:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574942.848940:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574942.848947:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574942.848952:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574942.848965:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574942.848967:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574942.848972:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574943.553011:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574943.553015:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574943.553019:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574943.872987:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574943.872993:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574943.872997:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574943.873003:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574943.873010:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574943.873014:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574943.873025:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574943.873029:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574943.873032:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574944.128957:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310303981, at 4310304000 00000100:00000010:0.0:1788574944.128979:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf120. 00000020:00000001:0.0:1788574944.577026:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574944.577030:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574944.577034:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574944.896951:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574944.896967:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574944.896969:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574944.896971:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574944.896974:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574944.896978:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574944.896993:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574944.896997:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574944.897001:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574945.920935:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574945.920940:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574945.920944:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574945.920951:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574945.920958:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574945.920964:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574945.920975:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574945.920978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574945.920981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574946.625006:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574946.625009:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574946.625011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574946.944976:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574946.944981:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574946.944984:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574946.944990:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574946.944996:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574946.945003:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574946.945007:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574946.945012:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574946.945018:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574947.969053:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574947.969057:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574947.969058:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574947.969062:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574947.969065:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574947.969069:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574947.969226:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574947.969228:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574947.969231:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574948.673054:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574948.673058:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574948.673070:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574948.992936:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574948.992944:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574948.992950:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574948.992961:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574948.992970:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574948.992991:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574948.993008:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574948.993011:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574948.993014:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574949.248895:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310309101, at 4310309120 00000100:00000010:0.0:1788574949.248906:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf780. 00000020:00000001:0.0:1788574949.696995:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574949.697000:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574949.697003:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574950.016906:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574950.016914:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574950.016917:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574950.016923:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574950.016927:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574950.016932:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574950.016944:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574950.016948:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574950.016951:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574951.040940:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574951.040949:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574951.040954:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574951.040962:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574951.040970:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574951.040977:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574951.040991:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574951.040996:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574951.041002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574951.041041:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574951.041046:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574951.041048:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574951.041051:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574951.041054:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574951.041056:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574951.041059:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574951.041061:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574951.041063:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574951.745052:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574951.745072:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574951.745077:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574952.064909:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574952.064912:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574952.064914:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574952.064918:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574952.064922:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574952.064926:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574952.064928:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574952.064931:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574952.064933:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574953.088947:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574953.088956:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574953.088960:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574953.088970:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574953.088977:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574953.088988:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574953.088995:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574953.089001:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574953.089012:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574953.089056:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574953.089061:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574953.089064:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574953.089066:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574953.089069:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574953.089071:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574953.089073:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574953.089074:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574953.089076:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574953.793100:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574953.793105:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574953.793109:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574954.112898:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574954.112903:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574954.112907:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574954.112914:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574954.112920:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574954.112923:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574954.112945:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574954.112947:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574954.112949:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574954.368932:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310314221, at 4310314240 00000100:00000010:0.0:1788574954.368952:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf5a0. 00000020:00000001:0.0:1788574954.817064:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574954.817067:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574954.817069:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574955.137075:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574955.137079:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574955.137081:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574955.137084:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574955.137086:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574955.137088:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574955.137092:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574955.137093:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574955.137097:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574955.648933:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574955.648948:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574955.648952:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574955.648956:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574955.648962:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574955.648970:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15643 00000400:00000200:0.0:1788574955.648985:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574955.648995:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574955.649000:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574955.649004:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649008:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574955.649011:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649013:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574955.649017:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574955.649018:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574955.649021:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574955.649045:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574955.649047:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574955.649049:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574955.649051:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649053:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574955.649054:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649055:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574955.649057:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574955.649060:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649062:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574955.649064:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574955.649067:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574955.649069:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.649071:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574955.649073:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574955.649079:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574955.649088:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574955.649092:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574955.649094:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.649097:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574955.649100:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574955.649105:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574955.649112:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574955.649120:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574955.649130:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516762112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574955.649139:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574955.649143:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574955.649147:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516762112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574955.649164:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574955.649168:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574955.649178:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574955.649179:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574955.649181:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574955.649184:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574955.649188:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15643 00000400:00000200:0.0:1788574955.649193:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574955.649200:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574955.649205:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574955.649212:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649218:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574955.649224:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649230:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574955.649235:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574955.649239:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574955.649243:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574955.649247:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574955.649251:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574955.649254:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574955.649259:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649262:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574955.649268:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649271:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574955.649278:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574955.649285:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649291:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574955.649294:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574955.649302:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574955.649306:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.649310:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574955.649317:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574955.649330:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.649334:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574955.649340:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574955.649344:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574955.649347:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574955.649350:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.649354:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574955.649356:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574955.649358:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574955.649366:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788574955.649373:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574955.649380:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516762240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574955.649393:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574955.649396:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574955.649403:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516762240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574955.649415:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574955.649419:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574955.649440:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574955.649441:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574955.649444:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574955.649447:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574955.649451:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15643 00000400:00000200:0.0:1788574955.649458:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574955.649462:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574955.649465:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574955.649470:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649473:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574955.649477:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649485:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574955.649487:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574955.649489:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574955.649492:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574955.649495:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574955.649498:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574955.649502:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574955.649509:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649513:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574955.649519:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649523:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574955.649527:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574955.649531:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649537:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574955.649541:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574955.649545:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574955.649548:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.649552:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574955.649555:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574955.649558:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.649560:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574955.649566:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574955.649570:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574955.649574:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574955.649577:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.649580:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574955.649593:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574955.649597:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574955.649604:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788574955.649610:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574955.649620:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516762368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574955.649647:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574955.649650:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574955.649659:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516762368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574955.649675:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574955.649681:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574955.649687:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574955.649689:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574955.649693:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574955.649700:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574955.649706:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15643 00000400:00000200:0.0:1788574955.649715:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574955.649723:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574955.649727:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574955.649735:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649740:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574955.649745:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649750:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574955.649754:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574955.649757:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574955.649760:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574955.649763:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574955.649768:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574955.649772:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574955.649779:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649783:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574955.649788:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649794:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574955.649806:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574955.649834:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574955.649839:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574955.649843:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574955.649848:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574955.649854:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.649859:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574955.649862:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574955.649868:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.649872:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574955.649877:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574955.649883:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574955.649887:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574955.649889:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.649895:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574955.649897:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574955.649901:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574955.649909:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788574955.649916:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574955.649926:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516762496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574955.649943:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574955.649947:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574955.649954:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516762496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574955.649969:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574955.649974:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574955.649979:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574955.649982:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574955.649985:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574955.649990:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574955.650001:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15643 00000400:00000200:0.0:1788574955.650006:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574955.650011:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574955.650013:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574955.650016:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.650018:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574955.650021:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.650024:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574955.650026:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574955.650028:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574955.650030:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574955.650032:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574955.650034:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574955.650038:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574955.650042:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.650044:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574955.650046:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574955.650049:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574955.650051:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574955.650055:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574955.650057:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574955.650060:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574955.650064:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574955.650067:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.650069:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574955.650072:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574955.650075:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.650080:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574955.650083:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574955.650084:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574955.650087:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574955.650094:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574955.650098:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574955.650101:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574955.650104:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574955.650110:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788574955.650116:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574955.650123:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516762624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574955.650140:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574955.650144:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574955.650150:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516762624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574955.650163:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574955.650167:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15653) 00000100:00000001:0.0:1788574956.160995:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574956.161002:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574956.161008:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574956.161017:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574956.161042:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574956.161048:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574956.161057:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574956.161062:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574956.161068:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574956.865023:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574956.865026:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574956.865028:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574957.184928:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574957.184935:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574957.184938:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574957.184944:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574957.184950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574957.184958:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574957.184965:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574957.184969:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574957.184974:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574958.208934:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574958.208938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574958.208942:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574958.208948:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574958.208955:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574958.208961:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574958.208979:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574958.208982:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574958.208986:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574958.913053:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574958.913058:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574958.913061:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574959.232987:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574959.232996:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574959.233004:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574959.233027:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574959.233042:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574959.233049:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574959.233059:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574959.233065:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574959.233072:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574959.488906:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310319341, at 4310319360 00000100:00000010:0.0:1788574959.488922:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf6c0. 00000020:00000001:0.0:1788574959.937034:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574959.937039:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574959.937042:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574960.257086:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574960.257090:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574960.257092:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574960.257095:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574960.257096:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574960.257099:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574960.257102:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574960.257104:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574960.257111:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574961.280984:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574961.280998:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574961.281030:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574961.281036:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574961.281044:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574961.281047:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574961.281061:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574961.281065:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574961.281071:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574961.985039:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574961.985042:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574961.985045:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574962.304916:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574962.304924:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574962.304929:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574962.304935:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574962.304940:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574962.304961:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574962.304974:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574962.304979:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574962.304982:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574963.328932:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574963.328939:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574963.328943:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574963.328948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574963.328955:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574963.328961:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574963.328966:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574963.328970:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574963.328975:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574964.033055:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574964.033059:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574964.033063:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574964.352927:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574964.352935:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574964.352939:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574964.352952:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574964.352960:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574964.352965:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574964.352969:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574964.352978:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574964.352981:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574964.608930:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310324461, at 4310324480 00000100:00000010:0.0:1788574964.608955:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfde0. 00000020:00000001:0.0:1788574965.056975:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574965.056979:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574965.056983:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574965.376949:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574965.376958:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574965.376960:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574965.376969:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574965.376976:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574965.376982:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574965.376988:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574965.376992:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574965.376997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574966.400959:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574966.400968:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574966.400973:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574966.400980:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574966.400989:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574966.400995:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574966.401007:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574966.401009:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574966.401012:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574967.105179:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574967.105183:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574967.105186:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574967.424972:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574967.424983:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574967.424989:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574967.424998:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574967.425014:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574967.425025:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574967.425050:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574967.425056:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574967.425062:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574968.448960:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574968.448967:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574968.448972:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574968.448992:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574968.449000:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574968.449007:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574968.449026:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574968.449031:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574968.449040:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574969.153055:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574969.153058:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574969.153061:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574969.472911:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574969.472915:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574969.472918:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574969.472923:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574969.472928:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574969.472933:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574969.472964:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574969.472969:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574969.472976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574969.728925:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310329581, at 4310329600 00000100:00000010:0.0:1788574969.728946:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfd80. 00000020:00000001:0.0:1788574970.177022:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574970.177027:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574970.177032:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574970.496928:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574970.496935:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574970.496939:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574970.496943:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574970.496946:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574970.496950:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574970.496954:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574970.496957:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574970.496966:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574971.008971:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574971.008982:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788574971.008999:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574971.009003:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574971.009009:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574971.009017:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15658 00000400:00000200:0.0:1788574971.009028:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574971.009035:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574971.009039:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574971.009043:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009048:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.009051:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009054:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574971.009059:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574971.009061:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788574971.009064:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574971.009067:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788574971.009069:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788574971.009072:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788574971.009076:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009078:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.009080:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009082:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574971.009085:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788574971.009102:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009105:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574971.009109:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574971.009112:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574971.009115:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009118:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788574971.009121:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574971.009127:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788574971.009131:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788574971.009142:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574971.009144:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009147:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574971.009150:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574971.009155:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574971.009163:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788574971.009170:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574971.009180:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516764032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574971.009190:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574971.009194:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574971.009199:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516764032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574971.009222:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574971.009226:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574971.009229:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788574971.009231:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574971.009233:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574971.009236:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574971.009238:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15658 00000400:00000200:0.0:1788574971.009251:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574971.009255:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574971.009258:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574971.009261:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009264:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.009266:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009269:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574971.009272:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574971.009273:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788574971.009275:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574971.009278:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788574971.009288:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788574971.009291:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788574971.009295:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009297:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.009300:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009302:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574971.009305:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788574971.009308:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009310:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574971.009312:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574971.009318:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574971.009321:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009324:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574971.009328:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574971.009331:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009333:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574971.009337:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788574971.009340:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574971.009343:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574971.009344:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009346:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574971.009348:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574971.009350:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574971.009354:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788574971.009358:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574971.009363:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516764160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574971.009372:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574971.009375:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574971.009379:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516764160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574971.009398:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574971.009402:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574971.009406:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788574971.009407:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574971.009409:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574971.009412:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574971.009415:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15658 00000400:00000200:0.0:1788574971.009420:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574971.009423:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574971.009426:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574971.009429:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009431:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.009434:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009436:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574971.009439:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574971.009440:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788574971.009442:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574971.009444:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788574971.009452:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788574971.009455:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788574971.009458:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009461:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.009463:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009466:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574971.009473:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788574971.009477:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009479:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574971.009486:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574971.009490:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574971.009492:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009494:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574971.009497:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788574971.009499:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009502:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574971.009505:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788574971.009508:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788574971.009510:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574971.009512:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009514:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574971.009516:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574971.009518:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574971.009522:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788574971.009525:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574971.009530:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516764288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574971.009539:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574971.009541:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574971.009551:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516764288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574971.009560:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574971.009564:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574971.009567:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788574971.009569:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574971.009570:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574971.009573:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574971.009577:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15658 00000400:00000200:0.0:1788574971.009581:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574971.009585:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574971.009587:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574971.009595:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009598:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.009600:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009602:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574971.009605:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574971.009607:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788574971.009612:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574971.009615:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788574971.009617:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788574971.009619:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788574971.009623:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009625:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.009627:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009630:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574971.009633:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788574971.009635:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009638:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574971.009640:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574971.009644:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574971.009646:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009649:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574971.009652:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574971.009654:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009656:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574971.009660:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788574971.009662:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574971.009665:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574971.009667:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009669:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574971.009670:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574971.009672:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574971.009676:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788574971.009684:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574971.009690:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516764416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574971.009699:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574971.009701:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574971.009706:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516764416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574971.009720:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788574971.009723:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788574971.009727:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788574971.009729:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788574971.009731:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788574971.009734:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788574971.009737:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15658 00000400:00000200:0.0:1788574971.009741:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788574971.009745:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788574971.009748:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788574971.009751:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009753:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.009756:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009758:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788574971.009760:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788574971.009762:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788574971.009764:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788574971.009766:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788574971.009769:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788574971.009771:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788574971.009775:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009777:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788574971.009784:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009787:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788574971.009790:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788574971.009793:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788574971.009796:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788574971.009798:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788574971.009801:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788574971.009804:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009806:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788574971.009830:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788574971.009833:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009835:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788574971.009839:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788574971.009842:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788574971.009845:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788574971.009846:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574971.009849:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788574971.009850:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788574971.009852:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788574971.009857:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788574971.009861:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574971.009867:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516764544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788574971.009875:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788574971.009877:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574971.009881:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516764544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788574971.009889:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574971.009895:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15668) 00000020:00000001:0.0:1788574972.225010:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574972.225014:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574972.225017:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574972.545018:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574972.545024:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574972.545029:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574972.545038:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574972.545046:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574972.545050:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574972.545055:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574972.545057:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574972.545060:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574973.568901:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574973.568905:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574973.568909:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574973.568916:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574973.568920:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574973.568924:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574973.568928:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574973.568931:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574973.568935:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574973.568963:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574973.568967:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574973.568970:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574973.568974:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574973.568979:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574973.568983:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574973.568996:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574973.568999:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574973.569001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574974.273029:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574974.273037:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574974.273039:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574974.592926:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574974.592933:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574974.592939:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574974.592946:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574974.592954:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574974.592960:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574974.592973:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574974.592977:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574974.593001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574974.848941:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310334702, at 4310334720 00000100:00000010:0.0:1788574974.848954:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfd20. 00000020:00000001:0.0:1788574975.297030:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574975.297033:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574975.297035:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574975.617018:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574975.617024:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574975.617028:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574975.617035:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574975.617041:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574975.617047:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574975.617053:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574975.617057:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574975.617061:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574976.640929:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574976.640938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574976.640942:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574976.640949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574976.640953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574976.640959:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574976.640965:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574976.640969:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574976.640972:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574976.641004:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574976.641008:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574976.641012:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574976.641018:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574976.641022:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574976.641025:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574976.641027:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574976.641029:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574976.641031:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574977.345048:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574977.345052:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574977.345054:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574977.664919:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574977.664939:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574977.664950:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574977.664962:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574977.664970:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574977.664977:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574977.664986:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574977.664990:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574977.664993:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574978.688914:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574978.688921:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574978.688924:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574978.688930:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574978.688935:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574978.688940:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574978.688947:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574978.688951:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574978.688957:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574978.688984:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574978.688987:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574978.688990:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574978.688992:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574978.688995:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574978.688997:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574978.689005:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574978.689006:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574978.689008:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574979.393018:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574979.393021:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574979.393024:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574979.712891:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574979.712898:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574979.712904:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574979.712913:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574979.712920:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574979.712925:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574979.712940:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574979.712946:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574979.712953:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574979.968893:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310339821, at 4310339840 00000100:00000010:0.0:1788574979.968909:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf8a0. 00000020:00000001:0.0:1788574980.416973:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574980.416976:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574980.416978:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574980.736933:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574980.736943:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574980.736947:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574980.736954:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574980.736961:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574980.736965:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574980.736985:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574980.736987:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574980.736990:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574981.760997:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574981.761001:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574981.761003:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574981.761007:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574981.761012:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574981.761016:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574981.761020:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574981.761023:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574981.761028:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574982.464970:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574982.464973:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574982.464976:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574982.784921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574982.784929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574982.784934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574982.784942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574982.784950:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574982.784955:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574982.784971:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574982.784974:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574982.784976:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574983.808898:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574983.808905:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574983.808925:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574983.808929:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574983.808935:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574983.808948:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574983.808962:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574983.808966:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574983.808970:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574984.513073:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574984.513080:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574984.513084:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574984.832918:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574984.832926:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574984.832929:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574984.832936:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574984.832943:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574984.832957:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574984.832963:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574984.832971:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574984.832981:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574984.833032:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574984.833037:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574984.833041:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574984.833045:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574984.833048:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574984.833051:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574984.833054:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574984.833055:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574984.833058:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574985.088966:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310344941, at 4310344960 00000100:00000010:0.0:1788574985.088993:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfd20. 00000020:00000001:0.0:1788574985.537037:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574985.537043:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574985.537048:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574985.857085:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574985.857094:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574985.857097:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574985.857103:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574985.857125:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574985.857128:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574985.857131:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574985.857134:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574985.857137:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574986.880937:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574986.880958:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574986.880962:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574986.880968:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574986.880973:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574986.880980:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574986.880987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574986.880991:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574986.880998:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574987.585014:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574987.585018:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574987.585022:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574987.904947:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574987.904956:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574987.904961:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574987.904969:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574987.904984:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574987.904988:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574987.905013:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574987.905015:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574987.905018:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574988.928917:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574988.928924:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574988.928938:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574988.928944:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574988.928950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574988.928956:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574988.928980:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574988.928984:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574988.928987:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574989.633056:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574989.633060:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574989.633062:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574989.952928:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574989.952935:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574989.952939:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574989.952945:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574989.952951:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574989.952955:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574989.952959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574989.952964:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574989.952968:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574989.953004:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574989.953007:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574989.953009:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574989.953011:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574989.953013:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574989.953016:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574989.953018:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574989.953020:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574989.953025:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574990.208943:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310350061, at 4310350080 00000100:00000010:0.0:1788574990.208960:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfd80. 00000020:00000001:0.0:1788574990.657027:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574990.657032:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574990.657035:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574990.976903:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574990.976907:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574990.976909:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574990.976912:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574990.976915:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574990.976917:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574990.976923:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574990.976927:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574990.976933:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574992.000928:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574992.000938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574992.000942:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574992.000952:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788574992.000960:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574992.000981:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574992.000989:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574992.000993:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574992.000996:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574992.001029:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574992.001032:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574992.001034:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574992.001036:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574992.001038:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574992.001040:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574992.001042:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574992.001043:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574992.001045:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574992.705065:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574992.705067:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574992.705070:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574993.024907:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574993.024913:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574993.024918:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574993.024926:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574993.024929:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574993.024934:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574993.024940:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574993.024944:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574993.024951:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788574993.537171:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788574993.537179:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574993.537193:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788574993.537198:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788574993.537201:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574994.048954:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574994.048963:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574994.048967:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574994.048979:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574994.048987:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574994.048994:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574994.049005:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574994.049011:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574994.049030:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574994.753036:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574994.753041:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574994.753045:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574995.072979:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574995.072989:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574995.072995:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574995.073002:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574995.073013:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574995.073020:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574995.073045:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574995.073055:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574995.073068:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788574995.328892:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310355182, at 4310355200 00000100:00000010:0.0:1788574995.328907:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfde0. 00000020:00000001:0.0:1788574995.776989:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574995.776994:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574995.776997:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574996.096920:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574996.096926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574996.096930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574996.096937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574996.096943:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574996.096946:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574996.096948:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574996.096953:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574996.096959:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574996.096977:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574996.096980:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574996.096983:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574996.096988:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574996.096991:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574996.096995:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574996.097007:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574996.097010:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574996.097012:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574997.120970:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574997.120992:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574997.120995:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574997.121001:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788574997.121008:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574997.121012:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574997.121017:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574997.121019:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574997.121025:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574997.121052:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574997.121056:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574997.121059:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574997.121061:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574997.121064:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574997.121068:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574997.121070:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574997.121072:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574997.121075:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574997.824968:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574997.824972:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574997.824982:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788574998.144945:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574998.144953:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574998.144967:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574998.144973:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788574998.144977:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574998.144981:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574998.144997:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574998.145000:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574998.145003:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574998.145070:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574998.145075:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574998.145079:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574998.145084:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574998.145089:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574998.145097:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574998.145100:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574998.145102:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574998.145104:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.168955:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574999.168967:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574999.168970:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574999.168976:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788574999.168981:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574999.168986:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574999.168990:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574999.168992:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574999.168994:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788574999.169074:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788574999.169079:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788574999.169083:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788574999.169088:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788574999.169093:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788574999.169098:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574999.169105:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788574999.169109:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788574999.169112:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788574999.873064:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788574999.873068:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788574999.873072:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575000.192938:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575000.192944:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575000.192947:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575000.192952:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575000.192955:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575000.192958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575000.192985:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575000.192990:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575000.192995:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575000.448923:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310360301, at 4310360320 00000100:00000010:0.0:1788575000.448941:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf6c0. 00000020:00000001:0.0:1788575000.897027:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575000.897030:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575000.897033:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575001.217070:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575001.217075:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575001.217093:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575001.217097:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575001.217103:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575001.217106:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575001.217212:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575001.217216:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575001.217220:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575002.240929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575002.240936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575002.240939:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575002.240947:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575002.240953:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575002.240960:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575002.240973:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575002.240976:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575002.240981:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575002.945006:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575002.945009:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575002.945012:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575003.264915:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575003.264921:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575003.264923:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575003.264928:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575003.264934:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575003.264938:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575003.264940:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575003.264942:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575003.264949:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575004.288945:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575004.288954:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575004.288961:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575004.288970:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575004.288980:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575004.288986:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575004.289010:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575004.289015:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575004.289022:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575004.993020:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575004.993039:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575004.993042:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575005.312929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575005.312936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575005.312939:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575005.312944:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575005.312951:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575005.312956:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575005.312960:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575005.312964:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575005.312968:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575005.312987:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575005.312991:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575005.312995:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575005.313000:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575005.313006:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575005.313010:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575005.313014:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575005.313016:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575005.313019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575005.568905:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310365421, at 4310365440 00000100:00000010:0.0:1788575005.568921:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf5a0. 00000020:00000001:0.0:1788575006.017013:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575006.017019:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575006.017023:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575006.336916:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575006.336919:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575006.336924:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575006.336927:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575006.336935:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575006.336939:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575006.336947:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575006.336950:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575006.336957:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575006.848942:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575006.848952:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575006.848970:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575006.848974:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575006.848978:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575006.848985:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15694 00000400:00000200:0.0:1788575006.848994:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575006.849000:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575006.849003:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575006.849006:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849010:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575006.849012:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849015:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575006.849020:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575006.849021:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575006.849025:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575006.849028:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575006.849030:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575006.849033:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575006.849036:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849038:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575006.849040:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849042:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575006.849045:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575006.849049:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849051:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575006.849055:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575006.849058:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575006.849061:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.849064:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575006.849066:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575006.849072:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575006.849076:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575006.849085:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575006.849087:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.849090:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575006.849093:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575006.849098:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575006.849107:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575006.849114:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575006.849123:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516768512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575006.849134:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575006.849139:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575006.849143:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516768512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575006.849165:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575006.849169:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575006.849173:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575006.849174:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575006.849176:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575006.849179:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575006.849182:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15694 00000400:00000200:0.0:1788575006.849188:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575006.849192:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575006.849196:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575006.849200:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849205:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575006.849210:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849215:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575006.849218:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575006.849220:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575006.849221:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575006.849224:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575006.849235:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575006.849240:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575006.849249:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849253:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575006.849258:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849262:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575006.849266:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575006.849273:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849278:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575006.849283:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575006.849290:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575006.849296:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.849301:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575006.849308:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575006.849314:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.849317:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575006.849333:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575006.849339:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575006.849344:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575006.849346:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.849351:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575006.849352:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575006.849354:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575006.849359:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575006.849363:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575006.849373:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516768640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575006.849396:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575006.849402:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575006.849413:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516768640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575006.849446:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575006.849452:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575006.849460:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575006.849466:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575006.849470:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575006.849476:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575006.849484:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15694 00000400:00000200:0.0:1788575006.849501:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575006.849509:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575006.849514:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575006.849521:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849528:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575006.849534:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849539:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575006.849545:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575006.849549:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575006.849552:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575006.849557:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575006.849562:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575006.849568:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575006.849575:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849580:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575006.849589:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849594:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575006.849601:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575006.849610:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849616:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575006.849630:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575006.849638:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575006.849661:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.849664:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575006.849666:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575006.849669:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.849681:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575006.849691:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575006.849697:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575006.849703:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575006.849708:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.849713:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575006.849717:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575006.849721:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575006.849730:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575006.849737:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575006.849748:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516768768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575006.849769:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575006.849774:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575006.849783:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516768768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575006.849847:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575006.849852:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575006.849857:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575006.849862:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575006.849865:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575006.849871:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575006.849878:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15694 00000400:00000200:0.0:1788575006.849887:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575006.849895:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575006.849901:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575006.849917:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849924:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575006.849931:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849935:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575006.849940:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575006.849944:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575006.849950:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575006.849955:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575006.849961:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575006.849966:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575006.849973:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849978:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575006.849984:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.849989:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575006.849995:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575006.850002:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575006.850005:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575006.850010:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575006.850016:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575006.850022:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.850026:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575006.850028:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575006.850035:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.850039:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575006.850045:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575006.850051:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575006.850058:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575006.850062:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.850068:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575006.850072:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575006.850076:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575006.850083:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575006.850096:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575006.850109:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516768896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575006.850139:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575006.850144:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575006.850154:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516768896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575006.850178:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575006.850183:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575006.850193:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575006.850197:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575006.850201:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575006.850209:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575006.850215:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15694 00000400:00000200:0.0:1788575006.850231:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575006.850241:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575006.850248:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575006.850258:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.850265:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575006.850272:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.850277:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575006.850283:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575006.850287:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575006.850292:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575006.850297:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575006.850302:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575006.850308:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575006.850318:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.850323:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575006.850336:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575006.850342:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575006.850349:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575006.850358:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575006.850366:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575006.850368:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575006.850373:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575006.850379:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.850384:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575006.850389:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575006.850397:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.850402:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575006.850412:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575006.850419:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575006.850425:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575006.850428:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575006.850433:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575006.850437:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575006.850441:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575006.850449:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575006.850457:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575006.850469:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516769024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575006.850493:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575006.850499:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575006.850509:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516769024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575006.850531:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575006.850539:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15704) 00000100:00000001:0.0:1788575007.360925:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575007.360933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575007.360937:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575007.360964:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575007.360972:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575007.360978:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575007.360986:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575007.360989:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575007.360993:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575008.064965:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575008.064969:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575008.064972:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575008.384932:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575008.384938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575008.384943:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575008.384949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575008.384955:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575008.384960:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575008.384979:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575008.384984:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575008.384999:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575009.408927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575009.408937:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575009.408941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575009.408949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575009.408956:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575009.408962:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575009.408965:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575009.408970:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575009.408973:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575009.409011:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575009.409016:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575009.409018:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575009.409021:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575009.409025:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575009.409028:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575009.409030:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575009.409032:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575009.409034:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575010.112999:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575010.113004:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575010.113024:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575010.432926:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575010.432932:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575010.432934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575010.432939:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575010.432946:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575010.432951:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575010.432966:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575010.432970:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575010.432974:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575010.688926:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310370541, at 4310370560 00000100:00000010:0.0:1788575010.688941:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfae0. 00000020:00000001:0.0:1788575011.137009:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575011.137013:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575011.137016:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575011.457015:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575011.457021:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575011.457026:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575011.457031:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575011.457037:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575011.457042:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575011.457060:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575011.457064:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575011.457068:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575012.480922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575012.480930:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575012.480936:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575012.480943:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575012.480950:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575012.480955:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575012.480967:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575012.480971:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575012.480976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575013.185010:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575013.185013:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575013.185016:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575013.504901:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575013.504915:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575013.504919:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575013.504924:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575013.504930:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575013.504935:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575013.504942:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575013.504948:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575013.504956:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575014.528955:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575014.528960:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575014.528964:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575014.528971:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575014.528974:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575014.528978:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575014.528980:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575014.528984:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575014.528986:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575015.233002:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575015.233005:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575015.233008:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575015.552942:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575015.552950:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575015.552953:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575015.552959:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575015.552964:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575015.552969:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575015.552973:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575015.552977:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575015.552984:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575015.808940:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310375661, at 4310375680 00000100:00000010:0.0:1788575015.808957:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf3c0. 00000020:00000001:0.0:1788575016.257051:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575016.257055:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575016.257058:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575016.576991:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575016.576998:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575016.577004:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575016.577028:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575016.577032:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575016.577034:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575016.577037:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575016.577040:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575016.577046:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575017.600928:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575017.600934:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575017.600938:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575017.600945:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575017.600952:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575017.600957:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575017.600977:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575017.600981:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575017.600986:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575018.304955:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575018.304958:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575018.304961:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575018.624924:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575018.624933:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575018.624938:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575018.624947:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575018.624954:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575018.624958:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575018.624963:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575018.624967:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575018.624972:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575019.648917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575019.648927:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575019.648930:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575019.648952:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575019.648960:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575019.648967:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575019.648980:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575019.648982:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575019.648989:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575020.353066:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575020.353069:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575020.353088:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575020.672910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575020.672918:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575020.672922:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575020.672926:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575020.672930:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575020.672934:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575020.672938:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575020.672941:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575020.672947:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575020.928933:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310380781, at 4310380800 00000100:00000010:0.0:1788575020.928946:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf2a0. 00000020:00000001:0.0:1788575021.377026:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575021.377030:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575021.377033:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575021.696982:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575021.697001:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575021.697005:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575021.697011:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575021.697014:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575021.697017:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575021.697040:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575021.697045:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575021.697051:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575022.721083:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575022.721088:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575022.721091:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575022.721096:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575022.721100:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575022.721103:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575022.721109:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575022.721112:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575022.721116:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575023.425041:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575023.425045:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575023.425049:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575023.744957:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575023.744972:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575023.744977:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575023.744984:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575023.744992:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575023.744999:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575023.745014:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575023.745019:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575023.745022:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575024.768880:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575024.768885:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575024.768889:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575024.768894:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575024.768899:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575024.768903:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575024.768908:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575024.768910:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575024.768914:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575024.768942:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575024.768945:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575024.768949:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575024.768954:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575024.768959:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575024.768962:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575024.768964:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575024.768965:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575024.768966:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575025.472997:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575025.473001:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575025.473004:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575025.792947:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575025.792953:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575025.792957:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575025.792965:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575025.792973:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575025.792978:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575025.792985:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575025.792990:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575025.792995:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575026.048915:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310385901, at 4310385920 00000100:00000010:0.0:1788575026.048933:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cff00. 00000020:00000001:0.0:1788575026.497055:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575026.497060:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575026.497064:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575026.816919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575026.816927:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575026.816931:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575026.816937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575026.816945:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575026.816950:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575026.816956:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575026.816960:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575026.816965:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575026.816995:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575026.816998:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575026.817000:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575026.817004:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575026.817008:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575026.817011:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575026.817013:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575026.817015:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575026.817017:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575027.840901:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575027.840910:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575027.840914:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575027.840920:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575027.840927:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575027.840933:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575027.840946:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575027.840951:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575027.840955:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575028.545020:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575028.545024:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575028.545026:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575028.865075:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575028.865080:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575028.865091:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575028.865098:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575028.865104:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575028.865108:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575028.865112:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575028.865116:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575028.865119:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575029.889059:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575029.889064:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575029.889066:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575029.889076:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575029.889078:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575029.889081:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575029.889084:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575029.889085:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575029.889088:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575030.593015:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575030.593019:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575030.593023:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575030.912932:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575030.912941:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575030.912946:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575030.912953:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575030.912961:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575030.912967:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575030.912977:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575030.912980:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575030.912985:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575031.168934:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310391021, at 4310391040 00000100:00000010:0.0:1788575031.168958:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf420. 00000020:00000001:0.0:1788575031.617052:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575031.617055:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575031.617058:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575031.936893:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575031.936901:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575031.936904:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575031.936912:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575031.936933:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575031.936938:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575031.936941:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575031.936945:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575031.936948:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575032.449158:0:2391:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788575032.449168:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575032.449186:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575032.449194:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575032.449200:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575032.960941:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575032.960950:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575032.960955:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575032.960963:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575032.960970:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575032.960975:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575032.961001:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575032.961005:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575032.961009:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575033.665001:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575033.665004:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575033.665006:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575033.984939:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575033.984945:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575033.984949:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575033.984957:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575033.984965:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575033.984971:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575033.984980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575033.984983:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575033.984995:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575035.008916:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575035.008926:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575035.008930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575035.008937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575035.008944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575035.008947:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575035.008981:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575035.008993:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575035.008997:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575035.713026:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575035.713030:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575035.713033:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575036.032921:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575036.032931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575036.032935:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575036.032944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575036.032953:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575036.032962:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575036.032970:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575036.032974:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575036.032978:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575036.033017:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575036.033022:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575036.033026:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575036.033031:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575036.033036:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575036.033038:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575036.033040:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575036.033041:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575036.033044:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575036.288905:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310396141, at 4310396160 00000100:00000010:0.0:1788575036.288917:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfea0. 00000020:00000001:0.0:1788575036.737012:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575036.737015:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575036.737017:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575037.056966:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575037.056971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575037.056974:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575037.056977:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575037.056980:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575037.056982:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575037.056983:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575037.056985:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575037.056987:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575038.080897:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575038.080905:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575038.080908:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575038.080913:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575038.080920:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575038.080925:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575038.080940:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575038.080944:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575038.080950:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575038.784958:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575038.784962:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575038.784965:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575039.104902:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575039.104909:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575039.104913:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575039.104920:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575039.104926:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575039.104929:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575039.104951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575039.104954:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575039.104956:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575039.105017:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575039.105021:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575039.105023:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575039.105026:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575039.105031:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575039.105035:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575039.105040:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575039.105042:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575039.105043:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575040.128895:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575040.128903:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575040.128911:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575040.128917:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575040.128923:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575040.128928:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575040.128936:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575040.128939:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575040.128953:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575040.832974:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575040.832977:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575040.832979:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575041.153053:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575041.153063:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575041.153067:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575041.153084:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575041.153091:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575041.153095:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575041.153101:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575041.153106:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575041.153111:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575041.408964:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310401261, at 4310401280 00000100:00000010:0.0:1788575041.408987:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf720. 00000020:00000001:0.0:1788575041.857022:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575041.857028:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575041.857032:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575042.176949:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575042.176955:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575042.176958:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575042.176963:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575042.176968:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575042.176975:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575042.176989:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575042.176994:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575042.177002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575043.200907:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575043.200915:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575043.200920:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575043.200929:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575043.200935:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575043.200940:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575043.200950:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575043.200953:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575043.200956:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575043.905002:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575043.905020:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575043.905023:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575044.224924:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575044.224932:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575044.224938:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575044.224944:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575044.224951:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575044.224955:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575044.224985:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575044.224988:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575044.224992:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575045.248910:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575045.248917:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575045.248921:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575045.248927:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575045.248933:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575045.248938:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575045.248949:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575045.248953:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575045.248958:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575045.953021:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575045.953023:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575045.953025:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575046.272945:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575046.272951:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575046.272954:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575046.272960:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575046.272966:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575046.272971:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575046.272982:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575046.272985:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575046.272988:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575046.528886:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310406382, at 4310406400 00000100:00000010:0.0:1788575046.528903:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf960. 00000020:00000001:0.0:1788575046.977010:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575046.977014:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575046.977018:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575047.296926:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575047.296931:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575047.296934:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575047.296938:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575047.296947:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575047.296949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575047.296951:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575047.296956:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575047.296958:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575047.297288:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575047.297292:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575047.297293:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575047.297295:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575047.297297:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575047.297299:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575047.297300:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575047.297301:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575047.297302:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575048.320909:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575048.320918:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575048.320923:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575048.320931:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575048.320937:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575048.320940:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575048.320945:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575048.320948:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575048.320951:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575049.024985:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575049.024988:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575049.024991:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575049.344904:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575049.344911:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575049.344915:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575049.344921:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575049.344928:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575049.344933:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575049.344940:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575049.344943:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575049.344959:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575050.368931:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575050.368936:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575050.368943:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575050.368950:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575050.368956:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575050.368961:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575050.368974:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575050.368978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575050.368982:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575051.072976:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575051.072979:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575051.072981:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575051.392913:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575051.392920:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575051.392924:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575051.392931:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575051.392937:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575051.392942:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575051.392947:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575051.392951:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575051.392955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575051.648910:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310411501, at 4310411520 00000100:00000010:0.0:1788575051.648927:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfe40. 00000020:00000001:0.0:1788575052.097061:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575052.097065:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575052.097069:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575052.416951:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575052.416961:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575052.416964:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575052.416971:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575052.416978:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575052.416984:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575052.416988:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575052.416992:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575052.416996:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575052.417030:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575052.417042:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575052.417047:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575052.417051:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575052.417055:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575052.417058:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575052.417060:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575052.417062:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575052.417063:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575053.440944:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575053.440953:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575053.440956:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575053.440965:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575053.440972:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575053.440978:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575053.440981:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575053.440986:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575053.440991:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575053.441007:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575053.441011:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575053.441013:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575053.441016:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575053.441020:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575053.441022:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575053.441024:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575053.441026:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575053.441029:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575054.145035:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575054.145041:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575054.145045:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575054.464964:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575054.464974:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575054.464977:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575054.464984:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575054.464992:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575054.464998:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575054.465006:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575054.465010:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575054.465016:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575056.193004:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575056.193008:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575056.193012:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575056.512922:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575056.512926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575056.512930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575056.512937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575056.512941:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575056.512945:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575056.512948:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575056.512951:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575056.512953:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575056.768909:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310416621, at 4310416640 00000100:00000010:0.0:1788575056.768922:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf300. 00000020:00000001:0.0:1788575057.217042:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575057.217045:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575057.217048:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575057.536946:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575057.536951:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575057.536958:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575057.536967:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575057.536973:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575057.536979:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575057.536986:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575057.536990:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575057.536993:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575058.561000:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575058.561005:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575058.561008:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575058.561012:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575058.561016:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575058.561019:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575058.561022:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575058.561025:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575058.561028:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575059.265047:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575059.265052:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575059.265076:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575059.584920:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575059.584926:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575059.584929:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575059.584937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575059.584943:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575059.584947:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575059.584968:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575059.584971:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575059.584973:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575060.608932:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575060.608938:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575060.608942:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575060.608950:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575060.608958:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575060.608963:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575060.608970:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575060.608973:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575060.608979:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575060.609020:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575060.609023:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575060.609024:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575060.609026:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575060.609029:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575060.609031:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575060.609032:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575060.609034:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575060.609036:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575061.313026:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575061.313029:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575061.313031:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575061.632917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575061.632924:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575061.632930:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575061.632939:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575061.632947:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575061.632954:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575061.632975:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575061.632986:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575061.632989:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575061.633022:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575061.633028:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575061.633032:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575061.633035:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575061.633038:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575061.633040:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575061.633043:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575061.633045:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575061.633048:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575061.888947:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310421741, at 4310421760 00000100:00000010:0.0:1788575061.888961:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfba0. 00000020:00000001:0.0:1788575062.337039:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575062.337042:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575062.337044:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575062.656979:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575062.656989:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575062.656994:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575062.657003:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575062.657009:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575062.657014:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575062.657036:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575062.657041:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575062.657049:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575063.680929:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575063.680942:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575063.680947:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575063.680968:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575063.680976:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575063.680982:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575063.681001:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575063.681005:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575063.681009:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575064.385001:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575064.385005:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575064.385008:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575064.704919:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575064.704924:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575064.704927:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575064.704934:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575064.704941:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575064.704947:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575064.704953:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575064.704956:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575064.704958:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575064.704996:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575064.705000:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575064.705004:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575064.705007:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575064.705013:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575064.705016:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575064.705017:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575064.705019:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575064.705021:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575065.728929:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575065.728938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575065.728943:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575065.728954:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575065.728961:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575065.728966:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575065.728971:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575065.728976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575065.728981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575065.729011:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575065.729016:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575065.729020:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575065.729024:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575065.729028:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575065.729030:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575065.729032:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575065.729034:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575065.729036:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575066.433089:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575066.433094:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575066.433114:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575066.752928:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575066.752933:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575066.752937:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575066.752943:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575066.752950:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575066.752956:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575066.752960:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575066.752963:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575066.752966:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575067.008914:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310426862, at 4310426880 00000100:00000010:0.0:1788575067.008928:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfc00. 00000020:00000001:0.0:1788575067.457004:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575067.457008:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575067.457011:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575067.776937:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575067.776946:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575067.776950:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575067.776959:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575067.776967:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575067.776974:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575067.776996:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575067.776999:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575067.777003:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575068.800908:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575068.800915:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575068.800919:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575068.800926:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575068.800931:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575068.800935:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575068.800938:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575068.800942:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575068.800949:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575069.505023:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575069.505027:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575069.505030:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575069.824928:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575069.824953:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575069.824957:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575069.824963:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575069.824969:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575069.824974:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575069.824989:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575069.824992:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575069.824997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575070.848912:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575070.848920:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575070.848926:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575070.848935:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575070.848942:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575070.848951:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575070.848967:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575070.848972:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575070.848976:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575071.361150:0:2391:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788575071.553026:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575071.553028:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575071.553031:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575071.872963:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575071.872971:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575071.872975:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575071.872982:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575071.872989:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575071.872994:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575071.872998:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575071.873002:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575071.873009:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575072.128947:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310431981, at 4310432000 00000100:00000010:0.0:1788575072.128964:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf7e0. 00000020:00000001:0.0:1788575072.577018:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575072.577021:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575072.577023:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575072.896914:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575072.896920:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575072.896946:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575072.896950:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575072.896956:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575072.896961:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575072.896966:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575072.896970:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575072.896974:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575073.920940:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575073.920955:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575073.920962:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575073.920969:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575073.920975:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575073.920981:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575073.920992:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575073.920996:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575073.921000:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575073.921033:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575073.921037:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575073.921039:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575073.921041:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575073.921045:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575073.921047:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575073.921050:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575073.921051:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575073.921054:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575074.625034:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575074.625039:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575074.625042:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575074.944936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575074.944944:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575074.944948:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575074.944953:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575074.944960:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575074.944965:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575074.944969:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575074.944974:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575074.944980:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575075.969014:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575075.969029:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575075.969031:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575075.969034:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575075.969036:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575075.969039:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575075.969040:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575075.969042:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575075.969046:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575076.673088:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575076.673093:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575076.673096:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575076.992950:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575076.992962:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575076.992966:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575076.992974:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575076.992982:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575076.992986:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575076.992992:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575076.992995:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575076.993001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575077.248951:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310437101, at 4310437120 00000100:00000010:0.0:1788575077.248967:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf480. 00000020:00000001:0.0:1788575077.697040:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575077.697043:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575077.697046:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575078.016924:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575078.016930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575078.016933:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575078.016940:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575078.016946:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575078.016952:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575078.016959:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575078.016963:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575078.016969:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575078.528900:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575078.528911:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575078.528934:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575078.528938:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575078.528942:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575078.528949:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15765 00000400:00000200:0.0:1788575078.528959:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575078.528965:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575078.528967:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575078.528969:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.528972:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575078.528974:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.528976:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575078.528980:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575078.528981:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575078.528984:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575078.528986:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575078.528988:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575078.528989:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575078.528991:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.528993:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575078.528994:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.528995:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575078.528997:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575078.529000:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529002:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575078.529004:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575078.529007:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575078.529009:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529011:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575078.529012:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575078.529019:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575078.529027:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575078.529035:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575078.529036:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529039:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575078.529042:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575078.529047:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575078.529054:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575078.529062:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575078.529070:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516777472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575078.529078:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575078.529082:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575078.529086:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516777472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575078.529102:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575078.529104:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575078.529107:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575078.529108:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575078.529109:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575078.529111:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575078.529113:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15765 00000400:00000200:0.0:1788575078.529115:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575078.529117:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575078.529118:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575078.529121:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529123:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575078.529124:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529126:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575078.529128:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575078.529128:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575078.529129:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575078.529131:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575078.529136:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575078.529138:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575078.529140:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529142:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575078.529143:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529145:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575078.529148:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575078.529149:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529150:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575078.529151:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575078.529157:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575078.529161:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529162:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575078.529166:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575078.529169:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529172:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575078.529176:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575078.529180:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575078.529184:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575078.529187:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529191:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575078.529194:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575078.529198:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575078.529205:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575078.529211:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575078.529221:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516777600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575078.529232:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575078.529235:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575078.529242:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516777600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575078.529274:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575078.529277:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575078.529281:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575078.529282:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575078.529285:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575078.529294:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575078.529299:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15765 00000400:00000200:0.0:1788575078.529304:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575078.529308:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575078.529311:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575078.529317:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529322:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575078.529327:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529331:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575078.529335:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575078.529337:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575078.529340:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575078.529343:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575078.529345:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575078.529349:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575078.529355:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529358:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575078.529360:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529361:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575078.529366:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575078.529368:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529370:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575078.529384:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575078.529388:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575078.529392:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529393:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575078.529396:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575078.529399:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529402:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575078.529406:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575078.529410:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575078.529415:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575078.529418:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529421:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575078.529424:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575078.529426:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575078.529431:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575078.529436:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575078.529443:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516777728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575078.529454:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575078.529456:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575078.529460:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516777728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575078.529466:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575078.529469:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575078.529473:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575078.529476:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575078.529477:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575078.529480:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575078.529484:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15765 00000400:00000200:0.0:1788575078.529489:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575078.529492:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575078.529495:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575078.529506:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529509:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575078.529513:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529514:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575078.529515:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575078.529516:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575078.529518:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575078.529521:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575078.529524:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575078.529527:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575078.529532:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529533:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575078.529534:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529539:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575078.529543:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575078.529545:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529548:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575078.529550:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575078.529554:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575078.529563:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529566:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575078.529569:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575078.529573:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529577:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575078.529580:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575078.529584:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575078.529589:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575078.529591:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529595:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575078.529598:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575078.529600:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575078.529605:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575078.529614:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575078.529620:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516777856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575078.529628:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575078.529631:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575078.529634:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516777856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575078.529643:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575078.529646:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575078.529649:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575078.529654:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575078.529656:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575078.529659:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575078.529664:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15765 00000400:00000200:0.0:1788575078.529673:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575078.529681:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575078.529686:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575078.529693:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529698:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575078.529700:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529702:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575078.529706:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575078.529708:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575078.529710:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575078.529714:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575078.529718:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575078.529722:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575078.529725:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529729:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575078.529737:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529740:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575078.529745:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575078.529750:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575078.529753:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575078.529755:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575078.529757:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575078.529758:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529761:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575078.529763:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575078.529766:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529770:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575078.529774:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575078.529778:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575078.529783:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575078.529785:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575078.529788:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575078.529790:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575078.529793:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575078.529796:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575078.529801:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575078.529844:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516777984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575078.529855:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575078.529858:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575078.529864:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516777984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575078.529882:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575078.529891:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15776) 00000100:00000001:0.0:1788575079.040963:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575079.040972:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575079.040976:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575079.041008:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575079.041021:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575079.041029:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575079.041051:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575079.041059:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575079.041065:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575079.745058:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575079.745062:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575079.745064:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575080.065059:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575080.065062:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575080.065063:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575080.065065:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575080.065066:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575080.065068:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575080.065069:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575080.065070:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575080.065072:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575080.065080:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575080.065081:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575080.065082:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575080.065084:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575080.065085:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575080.065086:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575080.065088:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575080.065089:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575080.065090:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575081.088903:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575081.088908:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575081.088910:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575081.088917:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575081.088925:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575081.088933:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575081.088938:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575081.088942:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575081.088946:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575081.793097:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575081.793102:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575081.793111:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575082.112929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575082.112938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575082.112943:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575082.112953:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575082.112960:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575082.112964:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575082.112969:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575082.112973:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575082.112976:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575082.112989:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575082.112995:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575082.112998:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575082.113002:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575082.113009:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575082.113013:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575082.113018:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575082.113020:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575082.113023:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575082.368925:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310442221, at 4310442240 00000100:00000010:0.0:1788575082.368940:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf240. 00000020:00000001:0.0:1788575082.816996:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575082.816999:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575082.817002:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575083.648922:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575083.648943:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575083.648971:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575083.648975:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575083.648980:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575083.648986:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15771 00000400:00000200:0.0:1788575083.648994:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575083.648998:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575083.649001:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575083.649003:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649022:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575083.649024:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649026:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575083.649029:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575083.649031:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575083.649034:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575083.649036:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575083.649038:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575083.649040:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575083.649042:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649043:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575083.649044:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649045:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575083.649047:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575083.649050:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649052:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575083.649054:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575083.649063:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575083.649066:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.649067:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575083.649069:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575083.649075:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575083.649078:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575083.649081:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575083.649083:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.649085:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575083.649088:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575083.649092:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575083.649099:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575083.649106:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575083.649116:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516778112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575083.649140:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575083.649145:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575083.649151:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516778112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575083.649172:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575083.649175:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575083.649179:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575083.649180:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575083.649182:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575083.649186:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575083.649190:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15771 00000400:00000200:0.0:1788575083.649195:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575083.649200:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575083.649206:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575083.649212:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649217:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575083.649222:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649228:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575083.649234:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575083.649236:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575083.649241:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575083.649246:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575083.649251:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575083.649256:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575083.649264:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649270:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575083.649276:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649282:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575083.649302:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575083.649314:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649319:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575083.649322:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575083.649329:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575083.649362:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.649368:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575083.649374:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575083.649382:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.649388:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575083.649399:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575083.649405:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575083.649410:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575083.649414:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.649420:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575083.649424:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575083.649428:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575083.649437:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575083.649446:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575083.649458:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516778240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575083.649479:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575083.649484:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575083.649493:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516778240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575083.649514:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575083.649520:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575083.649529:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575083.649533:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575083.649537:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575083.649541:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575083.649558:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15771 00000400:00000200:0.0:1788575083.649567:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575083.649577:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575083.649582:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575083.649590:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649595:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575083.649601:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649606:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575083.649611:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575083.649615:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575083.649617:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575083.649620:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575083.649625:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575083.649631:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575083.649639:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649643:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575083.649647:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649650:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575083.649656:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575083.649662:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649666:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575083.649670:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575083.649675:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575083.649681:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.649693:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575083.649696:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575083.649699:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.649702:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575083.649706:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575083.649710:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575083.649713:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575083.649720:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.649724:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575083.649728:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575083.649729:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575083.649734:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575083.649740:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575083.649747:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516778368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575083.649768:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575083.649772:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575083.649782:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516778368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575083.649828:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575083.649831:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575083.649834:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575083.649836:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575083.649838:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575083.649843:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575083.649850:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15771 00000400:00000200:0.0:1788575083.649860:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575083.649868:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575083.649873:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575083.649882:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649887:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575083.649893:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649899:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575083.649905:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575083.649909:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575083.649913:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575083.649917:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575083.649923:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575083.649935:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575083.649946:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649952:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575083.649957:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649963:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575083.649970:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575083.649976:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575083.649980:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575083.649983:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575083.649987:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575083.649990:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.649995:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575083.649998:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575083.650002:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.650004:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575083.650007:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575083.650008:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575083.650012:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575083.650014:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.650018:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575083.650019:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575083.650021:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575083.650025:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575083.650034:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575083.650045:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516778496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575083.650064:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575083.650070:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575083.650081:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516778496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575083.650101:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575083.650113:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575083.650121:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575083.650125:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575083.650128:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575083.650134:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575083.650141:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15771 00000400:00000200:0.0:1788575083.650150:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575083.650156:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575083.650162:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575083.650168:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.650170:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575083.650172:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.650176:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575083.650178:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575083.650181:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575083.650184:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575083.650187:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575083.650190:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575083.650195:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575083.650201:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.650204:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575083.650208:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575083.650212:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575083.650216:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575083.650222:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575083.650228:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575083.650231:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575083.650236:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575083.650241:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.650244:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575083.650254:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575083.650259:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.650265:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575083.650272:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575083.650279:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575083.650287:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575083.650290:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575083.650296:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575083.650300:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575083.650305:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575083.650312:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575083.650320:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575083.650347:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516778624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575083.650361:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575083.650366:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575083.650375:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516778624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575083.650400:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575083.650410:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15781) 00000100:00000001:0.0:1788575084.160976:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575084.160981:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575084.160984:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575084.160987:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575084.160990:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575084.160992:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575084.160994:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575084.160996:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575084.161000:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575084.865042:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575084.865046:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575084.865048:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575085.185025:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575085.185033:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575085.185044:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575085.185049:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575085.185052:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575085.185057:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575085.185069:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575085.185073:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575085.185076:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575086.208989:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575086.208997:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575086.209000:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575086.209004:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575086.209008:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575086.209012:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575086.209017:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575086.209021:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575086.209026:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575086.913035:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575086.913039:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575086.913041:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575087.232964:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575087.232971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575087.232973:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575087.232977:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575087.232982:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575087.232986:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575087.232991:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575087.232996:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575087.233000:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575087.488887:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310447341, at 4310447360 00000100:00000010:0.0:1788575087.488901:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf480. 00000020:00000001:0.0:1788575087.937011:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575087.937015:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575087.937018:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575088.256943:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575088.256947:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575088.256949:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575088.257101:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575088.257110:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575088.257113:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575088.257115:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575088.257117:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575088.257119:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575089.280930:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575089.280939:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575089.280945:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575089.280952:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575089.280959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575089.280963:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575089.280968:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575089.280972:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575089.280977:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575089.281005:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575089.281008:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575089.281012:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575089.281018:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575089.281023:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575089.281027:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575089.281042:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575089.281044:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575089.281046:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575089.984997:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575089.985000:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575089.985003:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575090.304962:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575090.304971:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575090.304976:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575090.304989:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575090.304994:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575090.304998:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575090.305033:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575090.305042:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575090.305061:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575091.328924:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575091.328933:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575091.328938:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575091.328957:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575091.328963:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575091.328969:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575091.328983:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575091.328988:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575091.328991:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575092.033057:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575092.033060:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575092.033062:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575092.352925:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575092.352930:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575092.352934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575092.352941:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575092.352948:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575092.352954:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575092.352972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575092.352976:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575092.352980:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575092.352998:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575092.353002:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575092.353004:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575092.353006:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575092.353010:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575092.353012:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575092.353015:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575092.353016:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575092.353019:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575092.608898:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310452461, at 4310452480 00000100:00000010:0.0:1788575092.608914:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf7e0. 00000020:00000001:0.0:1788575093.056961:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575093.056965:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575093.056968:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575093.376980:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575093.376985:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575093.376990:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575093.376999:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575093.377010:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575093.377039:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575093.377052:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575093.377056:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575093.377061:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575094.400942:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575094.400950:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575094.400955:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575094.400959:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575094.400969:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575094.400975:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575094.400990:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575094.400994:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575094.401000:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575095.104975:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575095.104980:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575095.104983:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575095.424932:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575095.424941:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575095.424944:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575095.424952:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575095.424962:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575095.424967:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575095.424976:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575095.424980:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575095.424989:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575095.425022:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575095.425028:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575095.425032:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575095.425038:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575095.425043:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575095.425049:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575095.425058:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575095.425062:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575095.425065:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575096.448960:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575096.448974:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575096.448979:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575096.448994:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575096.449016:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575096.449023:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575096.449035:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575096.449040:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575096.449045:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575097.152993:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575097.152995:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575097.152998:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575097.472903:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575097.472911:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575097.472914:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575097.472921:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575097.472928:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575097.472933:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575097.472945:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575097.472948:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575097.472951:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575097.728910:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310457581, at 4310457600 00000100:00000010:0.0:1788575097.728927:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfc00. 00000020:00000001:0.0:1788575098.176985:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575098.176988:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575098.176991:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575098.496930:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575098.496940:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575098.496945:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575098.496950:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575098.496956:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575098.496961:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575098.496966:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575098.496969:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575098.496974:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575099.520966:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575099.520985:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575099.520990:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575099.520996:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575099.521003:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575099.521008:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575099.521040:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575099.521050:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575099.521055:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575100.225048:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575100.225055:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575100.225059:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575100.544991:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575100.545001:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575100.545005:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575100.545014:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575100.545020:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575100.545026:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575100.545047:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575100.545053:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575100.545058:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575101.568962:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575101.568971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575101.568976:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575101.568983:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575101.568989:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575101.568994:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575101.569004:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575101.569006:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575101.569009:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788575102.081084:0:2388:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788575102.081123:0:2388:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788575102.081132:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575102.081136:0:2388:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788575102.081141:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575102.081144:0:2388:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788575102.081149:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575102.081153:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793000. 00000400:00000200:0.0:1788575102.081158:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575102.081163:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575102.081169:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc3b8 00000400:00000010:0.0:1788575102.081184:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc3b8. 00000100:00000001:0.0:1788575102.081194:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575102.081203:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac820700 x1875440516780032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575102.081232:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575102.081237:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516780032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575102.081245:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575102.081247:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575102.081257:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575102.081259:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793b00. 00000400:00000200:0.0:1788575102.081262:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575102.081265:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575102.081268:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcee0 00000400:00000010:0.0:1788575102.081270:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcee0. 00000100:00000001:0.0:1788575102.081273:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575102.081276:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac822300 x1875440516780160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575102.081284:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575102.081287:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516780160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575102.081293:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575102.081295:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575102.081300:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575102.081302:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793400. 00000400:00000200:0.0:1788575102.081305:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575102.081308:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575102.081310:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fca18 00000400:00000010:0.0:1788575102.081312:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fca18. 00000100:00000001:0.0:1788575102.081314:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575102.081318:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac823480 x1875440516780288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575102.081341:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575102.081344:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516780288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575102.081351:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575102.081353:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575102.081359:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575102.081361:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793d00. 00000400:00000200:0.0:1788575102.081364:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575102.081367:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575102.081371:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc198 00000400:00000010:0.0:1788575102.081372:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc198. 00000100:00000001:0.0:1788575102.081374:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575102.081378:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac821f80 x1875440516780416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575102.081385:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575102.081387:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516780416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575102.081392:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575102.081393:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575102.081397:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575102.081399:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793500. 00000400:00000200:0.0:1788575102.081402:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575102.081404:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575102.081406:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc550 00000400:00000010:0.0:1788575102.081408:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc550. 00000100:00000001:0.0:1788575102.081409:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575102.081412:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac822d80 x1875440516780544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575102.081418:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575102.081420:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516780544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575102.081431:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575102.081433:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788575102.273137:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575102.273143:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575102.273146:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575102.592924:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575102.592933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575102.592936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575102.592941:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575102.592951:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575102.592956:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575102.592958:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575102.592960:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575102.592963:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575102.848891:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310462701, at 4310462720 00000100:00000010:0.0:1788575102.848907:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfba0. 00000020:00000001:0.0:1788575103.296964:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575103.296968:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575103.296971:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575103.616937:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575103.616942:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575103.616943:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575103.616949:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575103.616955:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575103.616960:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575103.616967:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575103.616971:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575103.616974:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575103.617003:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575103.617008:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575103.617012:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575103.617015:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575103.617017:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575103.617019:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575103.617021:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575103.617031:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575103.617033:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575104.640901:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575104.640906:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575104.640909:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575104.640914:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575104.640920:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575104.640923:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575104.640926:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575104.640930:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575104.640933:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575104.640947:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575104.640950:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575104.640954:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575104.640956:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575104.640962:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575104.640965:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575104.640967:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575104.640968:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575104.640970:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575105.344956:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575105.344960:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575105.344963:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575105.664908:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575105.664916:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575105.664921:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575105.664929:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575105.664936:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575105.664943:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575105.664953:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575105.664956:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575105.664960:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575106.688902:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575106.688909:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575106.688911:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575106.688917:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575106.688922:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575106.688926:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575106.688952:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575106.688954:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575106.688957:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575107.393002:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575107.393005:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575107.393007:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575107.968913:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310467822, at 4310467840 00000100:00000010:0.0:1788575107.968933:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf300. 00000020:00000001:0.0:1788575108.417008:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575108.417011:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575108.417013:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575108.736926:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575108.736932:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575108.736936:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575108.736943:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575108.736950:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575108.736955:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575108.736974:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575108.736977:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575108.736981:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575109.248897:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575109.248904:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575109.248907:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575109.248911:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575109.248915:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575109.248921:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15796 00000400:00000200:0.0:1788575109.248929:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575109.248934:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575109.248937:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575109.248939:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.248942:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575109.248944:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.248946:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575109.248948:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575109.248963:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575109.248966:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575109.248968:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575109.248970:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575109.248972:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575109.248974:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.248976:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575109.248977:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.248979:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575109.248981:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575109.248983:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575109.248984:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575109.248986:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575109.248989:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575109.248990:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.248992:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575109.248993:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575109.248998:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575109.249000:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575109.249003:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575109.249004:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249006:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575109.249008:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575109.249011:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575109.249016:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575109.249021:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575109.249028:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516781312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575109.249035:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575109.249038:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575109.249042:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516781312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575109.249064:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575109.249066:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575109.249069:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575109.249072:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575109.249074:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575109.249078:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575109.249080:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15796 00000400:00000200:0.0:1788575109.249084:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575109.249088:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575109.249090:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575109.249095:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249097:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575109.249102:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249105:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575109.249107:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575109.249109:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575109.249111:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575109.249113:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575109.249116:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575109.249120:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575109.249125:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249128:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575109.249132:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249134:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575109.249138:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575109.249140:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249144:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575109.249151:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575109.249156:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575109.249166:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249169:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575109.249175:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575109.249178:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249179:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575109.249182:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575109.249185:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575109.249189:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575109.249190:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249192:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575109.249193:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575109.249195:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575109.249199:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575109.249201:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575109.249208:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516781440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575109.249216:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575109.249218:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575109.249224:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516781440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575109.249230:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575109.249233:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575109.249237:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575109.249238:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575109.249240:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575109.249243:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575109.249246:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15796 00000400:00000200:0.0:1788575109.249251:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575109.249256:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575109.249259:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575109.249270:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249275:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575109.249278:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249281:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575109.249283:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575109.249285:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575109.249288:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575109.249290:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575109.249292:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575109.249301:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575109.249305:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249307:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575109.249309:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249310:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575109.249316:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575109.249321:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249326:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575109.249330:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575109.249335:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575109.249340:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249350:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575109.249353:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575109.249359:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249364:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575109.249368:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575109.249372:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575109.249376:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575109.249378:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249382:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575109.249384:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575109.249387:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575109.249391:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575109.249399:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575109.249406:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516781568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575109.249414:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575109.249417:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575109.249423:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516781568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575109.249434:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575109.249438:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575109.249443:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575109.249446:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575109.249448:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575109.249450:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575109.249454:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15796 00000400:00000200:0.0:1788575109.249459:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575109.249465:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575109.249466:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575109.249469:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249472:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575109.249474:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249475:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575109.249476:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575109.249477:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575109.249479:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575109.249482:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575109.249483:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575109.249485:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575109.249494:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249497:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575109.249506:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249508:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575109.249511:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575109.249515:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249519:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575109.249522:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575109.249525:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575109.249530:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249531:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575109.249534:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575109.249538:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249540:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575109.249543:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575109.249544:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575109.249548:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575109.249549:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249552:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575109.249554:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575109.249556:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575109.249560:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575109.249562:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575109.249568:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516781696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575109.249575:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575109.249577:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575109.249582:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516781696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575109.249591:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575109.249595:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575109.249598:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575109.249600:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575109.249606:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575109.249610:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575109.249613:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15796 00000400:00000200:0.0:1788575109.249620:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575109.249623:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575109.249627:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575109.249631:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249636:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575109.249639:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249643:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575109.249645:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575109.249647:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575109.249649:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575109.249653:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575109.249655:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575109.249659:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575109.249662:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249665:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575109.249668:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249670:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575109.249674:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575109.249678:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575109.249681:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575109.249684:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575109.249687:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575109.249691:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249694:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575109.249696:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575109.249698:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249700:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575109.249707:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575109.249709:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575109.249710:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575109.249711:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575109.249712:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575109.249712:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575109.249714:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575109.249715:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575109.249718:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575109.249722:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516781824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575109.249736:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575109.249739:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575109.249744:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516781824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575109.249759:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575109.249773:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15806) 00000100:00000001:0.0:1788575109.760890:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575109.760897:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575109.760900:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575109.760906:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575109.760912:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575109.760918:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575109.760931:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575109.760934:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575109.760938:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575110.464963:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575110.464966:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575110.464969:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575110.784916:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575110.784924:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575110.784928:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575110.784938:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575110.784944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575110.784950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575110.784975:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575110.784979:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575110.784984:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575111.808878:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575111.808887:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575111.808891:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575111.808899:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575111.808906:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575111.808912:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575111.808926:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575111.808930:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575111.808933:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575111.808962:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575111.808966:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575111.808969:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575111.808973:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575111.808976:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575111.808978:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575111.808980:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575111.808981:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575111.808983:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575112.512972:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575112.512975:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575112.512977:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575112.832917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575112.832923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575112.832927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575112.832935:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575112.832942:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575112.832947:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575112.832970:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575112.832973:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575112.832976:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575113.088922:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310472941, at 4310472960 00000100:00000010:0.0:1788575113.088937:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf900. 00000020:00000001:0.0:1788575113.536990:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575113.536994:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575113.537012:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575113.856925:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575113.856933:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575113.856936:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575113.856944:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575113.856951:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575113.856956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575113.856972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575113.856975:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575113.856977:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575114.368908:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575114.368919:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575114.368922:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575114.368926:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575114.368931:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575114.368938:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15801 00000400:00000200:0.0:1788575114.368947:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575114.368953:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575114.368957:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575114.368960:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.368964:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.368967:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.368969:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575114.368973:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575114.368975:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575114.368977:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575114.368980:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575114.368983:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575114.368985:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575114.368988:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.368990:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.368991:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369014:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575114.369017:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575114.369021:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369023:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575114.369026:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575114.369029:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575114.369032:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369034:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575114.369037:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575114.369042:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575114.369045:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575114.369049:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575114.369050:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369052:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575114.369054:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575114.369058:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575114.369070:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575114.369075:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575114.369082:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516781952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575114.369092:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575114.369094:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575114.369098:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516781952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575114.369111:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575114.369114:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575114.369116:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575114.369117:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575114.369118:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575114.369119:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575114.369121:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15801 00000400:00000200:0.0:1788575114.369128:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575114.369131:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575114.369133:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575114.369135:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369139:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.369143:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369146:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575114.369148:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575114.369150:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575114.369152:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575114.369155:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575114.369159:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575114.369162:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575114.369168:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369171:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.369174:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369178:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575114.369183:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575114.369187:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369191:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575114.369199:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575114.369205:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575114.369208:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369212:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575114.369217:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575114.369222:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369225:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575114.369231:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575114.369233:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575114.369235:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575114.369249:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369253:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575114.369255:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575114.369258:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575114.369266:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575114.369273:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575114.369280:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516782080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575114.369295:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575114.369298:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575114.369305:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516782080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575114.369320:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575114.369325:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575114.369332:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575114.369336:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575114.369339:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575114.369341:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575114.369347:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15801 00000400:00000200:0.0:1788575114.369365:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575114.369372:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575114.369376:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575114.369387:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369390:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.369394:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369398:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575114.369402:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575114.369404:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575114.369407:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575114.369410:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575114.369415:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575114.369426:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575114.369434:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369439:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.369445:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369451:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575114.369457:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575114.369464:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369467:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575114.369471:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575114.369474:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575114.369479:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369483:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575114.369486:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575114.369492:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369495:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575114.369498:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575114.369501:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575114.369506:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575114.369507:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369511:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575114.369513:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575114.369516:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575114.369528:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575114.369533:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575114.369540:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516782208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575114.369549:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575114.369552:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575114.369559:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516782208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575114.369584:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575114.369594:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575114.369602:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575114.369606:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575114.369609:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575114.369615:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575114.369619:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15801 00000400:00000200:0.0:1788575114.369624:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575114.369628:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575114.369632:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575114.369637:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369641:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.369645:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369647:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575114.369651:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575114.369654:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575114.369657:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575114.369660:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575114.369663:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575114.369666:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575114.369671:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369674:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.369683:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369687:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575114.369692:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575114.369697:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369701:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575114.369704:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575114.369707:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575114.369711:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369714:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575114.369722:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575114.369726:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369731:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575114.369736:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575114.369739:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575114.369744:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575114.369747:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369749:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575114.369751:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575114.369754:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575114.369757:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575114.369763:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575114.369769:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516782336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575114.369781:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575114.369784:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575114.369793:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516782336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575114.369842:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575114.369855:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575114.369862:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575114.369864:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575114.369868:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575114.369874:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575114.369880:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15801 00000400:00000200:0.0:1788575114.369890:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575114.369898:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575114.369902:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575114.369906:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369908:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.369911:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369919:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575114.369921:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575114.369922:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575114.369923:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575114.369924:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575114.369928:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575114.369929:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575114.369933:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369938:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575114.369941:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369945:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575114.369947:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575114.369952:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575114.369956:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575114.369958:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575114.369963:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575114.369968:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369972:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575114.369975:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575114.369978:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369981:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575114.369984:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575114.369988:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575114.369992:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575114.369995:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575114.369999:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575114.370001:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575114.370003:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575114.370010:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575114.370017:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575114.370027:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516782464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575114.370059:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575114.370063:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575114.370069:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516782464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575114.370083:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575114.370086:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15811) 00000100:00000001:0.0:1788575114.880947:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575114.880952:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575114.880955:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575114.880961:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575114.880967:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575114.880972:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575114.880991:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575114.880993:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575114.880995:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575115.585003:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575115.585005:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575115.585008:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575115.904921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575115.904926:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575115.904931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575115.904936:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575115.904940:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575115.904945:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575115.904949:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575115.904951:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575115.904954:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575116.928949:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575116.928953:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575116.928957:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575116.928963:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575116.928969:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575116.928973:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575116.928977:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575116.928998:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575116.929002:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575117.632998:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575117.633001:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575117.633004:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575117.952903:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575117.952935:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575117.952939:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575117.952946:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575117.952952:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575117.952955:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575117.952965:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575117.952969:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575117.952972:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575118.208924:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310478061, at 4310478080 00000100:00000010:0.0:1788575118.208938:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfa20. 00000020:00000001:0.0:1788575118.656995:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575118.656998:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575118.657000:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575118.976913:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575118.976920:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575118.976938:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575118.976943:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575118.976949:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575118.976953:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575118.976960:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575118.976963:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575118.976966:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575118.977058:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575118.977061:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575118.977063:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575118.977065:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575118.977067:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575118.977070:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575118.977072:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575118.977074:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575118.977078:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575118.977117:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575118.977120:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575118.977122:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575118.977124:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575118.977126:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575118.977128:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575118.977131:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575118.977132:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575118.977135:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575120.000929:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575120.000936:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575120.000952:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575120.000958:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575120.000964:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575120.000971:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575120.000987:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575120.000990:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575120.000994:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575120.705007:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575120.705012:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575120.705015:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575121.024927:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575121.024934:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575121.024937:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575121.024948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575121.024955:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575121.024976:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575121.024991:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575121.024997:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575121.025008:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575122.048928:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575122.048935:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575122.048936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575122.048941:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575122.048944:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575122.048948:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575122.048954:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575122.048958:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575122.048995:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575122.753013:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575122.753017:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575122.753020:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575123.072916:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575123.072923:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575123.072926:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575123.072932:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575123.072937:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575123.072942:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575123.072959:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575123.072963:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575123.072967:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575123.328913:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310483181, at 4310483200 00000100:00000010:0.0:1788575123.328931:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf360. 00000020:00000001:0.0:1788575123.777023:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575123.777025:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575123.777027:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575124.096902:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575124.096908:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575124.096913:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575124.096919:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575124.096925:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575124.096931:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575124.096942:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575124.096945:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575124.096949:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575125.120917:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575125.120924:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575125.120929:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575125.120936:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575125.120942:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575125.120948:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575125.120955:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575125.120960:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575125.120965:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575125.825025:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575125.825060:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575125.825063:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575126.145226:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575126.145235:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575126.145240:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575126.145246:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575126.145252:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575126.145257:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575126.145268:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575126.145273:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575126.145276:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575127.168917:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575127.168924:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575127.168927:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575127.168932:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575127.168938:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575127.168944:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575127.168951:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575127.168955:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575127.168959:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575127.873027:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575127.873030:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575127.873033:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575128.192959:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575128.192965:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575128.192967:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575128.192970:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575128.192972:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575128.192974:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575128.192988:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575128.192992:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575128.192995:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575128.448923:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310488301, at 4310488320 00000100:00000010:0.0:1788575128.448944:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfea0. 00000020:00000001:0.0:1788575128.896988:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575128.896992:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575128.896995:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575129.216950:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575129.216954:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575129.216955:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575129.216958:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575129.216961:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575129.216963:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575129.216966:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575129.216968:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575129.216970:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575129.728930:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575129.728940:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575129.728943:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575129.728947:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575129.728951:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575129.728958:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15817 00000400:00000200:0.0:1788575129.728966:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575129.728972:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575129.728974:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575129.728977:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.728980:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575129.728982:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.728984:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575129.728988:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575129.728990:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575129.728992:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575129.728994:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575129.728997:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575129.729000:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575129.729002:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729004:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575129.729006:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729008:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575129.729026:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575129.729030:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729032:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575129.729035:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575129.729038:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575129.729041:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.729043:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575129.729045:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575129.729052:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575129.729056:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575129.729059:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575129.729061:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.729063:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575129.729066:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575129.729070:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575129.729076:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575129.729083:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575129.729091:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516783872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575129.729101:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575129.729133:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575129.729138:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516783872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575129.729164:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575129.729168:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575129.729171:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575129.729173:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575129.729174:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575129.729177:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575129.729180:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15817 00000400:00000200:0.0:1788575129.729185:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575129.729195:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575129.729200:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575129.729210:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729216:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575129.729222:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729228:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575129.729234:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575129.729238:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575129.729243:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575129.729248:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575129.729253:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575129.729259:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575129.729268:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729273:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575129.729279:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729283:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575129.729289:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575129.729294:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729299:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575129.729304:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575129.729312:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575129.729318:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.729324:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575129.729331:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575129.729336:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.729340:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575129.729346:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575129.729353:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575129.729359:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575129.729364:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.729369:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575129.729379:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575129.729384:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575129.729392:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575129.729399:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575129.729410:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516784000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575129.729430:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575129.729435:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575129.729445:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516784000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575129.729465:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575129.729470:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575129.729480:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575129.729482:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575129.729484:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575129.729490:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575129.729496:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15817 00000400:00000200:0.0:1788575129.729503:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575129.729508:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575129.729511:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575129.729517:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729520:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575129.729524:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729530:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575129.729535:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575129.729539:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575129.729543:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575129.729546:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575129.729549:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575129.729553:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575129.729565:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729570:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575129.729575:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729579:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575129.729585:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575129.729590:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729596:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575129.729598:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575129.729603:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575129.729607:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.729610:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575129.729614:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575129.729620:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.729625:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575129.729631:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575129.729636:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575129.729640:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575129.729644:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.729649:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575129.729653:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575129.729657:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575129.729665:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575129.729672:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575129.729682:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516784128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575129.729697:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575129.729702:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575129.729710:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516784128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575129.729732:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575129.729737:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575129.729758:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575129.729763:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575129.729768:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575129.729773:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575129.729779:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15817 00000400:00000200:0.0:1788575129.729785:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575129.729792:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575129.729797:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575129.729804:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729853:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575129.729859:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729866:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575129.729872:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575129.729876:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575129.729880:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575129.729884:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575129.729889:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575129.729893:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575129.729901:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729906:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575129.729911:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729914:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575129.729918:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575129.729920:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575129.729925:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575129.729929:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575129.729934:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575129.729939:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.729943:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575129.729948:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575129.729959:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.729963:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575129.729969:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575129.729975:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575129.729981:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575129.729983:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.729988:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575129.729991:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575129.729995:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575129.730002:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575129.730009:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575129.730019:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516784256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575129.730036:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575129.730039:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575129.730051:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516784256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575129.730060:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575129.730063:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575129.730066:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575129.730067:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575129.730069:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575129.730071:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575129.730073:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15817 00000400:00000200:0.0:1788575129.730077:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575129.730079:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575129.730081:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575129.730084:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.730086:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575129.730088:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.730095:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575129.730098:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575129.730099:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575129.730100:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575129.730115:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575129.730117:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575129.730119:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575129.730122:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.730124:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575129.730126:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575129.730128:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575129.730130:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575129.730133:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575129.730135:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575129.730137:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575129.730140:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575129.730142:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.730144:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575129.730146:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575129.730148:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.730150:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575129.730155:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575129.730157:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575129.730159:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575129.730160:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575129.730161:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575129.730163:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575129.730165:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575129.730169:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575129.730174:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575129.730181:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516784384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575129.730209:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575129.730214:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575129.730224:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516784384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575129.730244:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575129.730253:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15827) 00000100:00000001:0.0:1788575130.240945:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575130.240949:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575130.240953:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575130.240957:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575130.240960:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575130.240963:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575130.240981:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575130.240983:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575130.240986:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575130.945035:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575130.945039:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575130.945043:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575131.264952:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575131.264958:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575131.264961:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575131.264966:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575131.264971:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575131.264976:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575131.264979:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575131.264981:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575131.264984:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575132.288963:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575132.288967:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575132.288971:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575132.288978:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575132.288984:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575132.288990:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575132.288994:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575132.289003:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575132.289006:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575132.992998:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575132.993001:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575132.993003:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575133.312956:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575133.312962:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575133.312965:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575133.312969:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575133.312972:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575133.312974:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575133.312988:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575133.312991:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575133.312995:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575133.568882:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310493421, at 4310493440 00000100:00000010:0.0:1788575133.568890:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf180. 00000020:00000001:0.0:1788575134.016962:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575134.016966:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575134.016969:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575134.336915:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575134.336922:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575134.336927:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575134.336934:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575134.336940:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575134.336946:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575134.336952:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575134.336954:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575134.336956:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575134.848899:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575134.848909:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575134.848912:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575134.848916:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575134.848921:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575134.848929:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15822 00000400:00000200:0.0:1788575134.848939:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575134.848968:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575134.848970:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575134.848973:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.848975:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575134.848977:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.848979:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575134.848982:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575134.848983:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575134.848985:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575134.848987:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575134.848989:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575134.848990:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575134.848992:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.848993:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575134.848994:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.848996:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575134.848998:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575134.849000:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849002:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575134.849004:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575134.849006:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575134.849008:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849009:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575134.849010:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575134.849016:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575134.849019:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575134.849021:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575134.849023:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849024:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575134.849026:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575134.849030:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575134.849035:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575134.849052:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575134.849059:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516784512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575134.849067:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575134.849069:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575134.849073:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516784512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575134.849087:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575134.849089:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575134.849092:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575134.849093:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575134.849094:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575134.849095:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575134.849097:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15822 00000400:00000200:0.0:1788575134.849099:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575134.849101:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575134.849102:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575134.849104:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849106:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575134.849107:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849108:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575134.849109:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575134.849110:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575134.849111:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575134.849112:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575134.849119:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575134.849120:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575134.849122:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849125:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575134.849132:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849133:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575134.849135:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575134.849136:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849139:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575134.849140:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575134.849145:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575134.849147:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849149:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575134.849154:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575134.849157:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849159:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575134.849162:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575134.849164:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575134.849166:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575134.849168:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849171:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575134.849173:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575134.849175:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575134.849179:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575134.849184:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575134.849202:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516784640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575134.849213:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575134.849216:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575134.849222:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516784640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575134.849234:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575134.849237:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575134.849240:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575134.849242:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575134.849249:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575134.849252:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575134.849254:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15822 00000400:00000200:0.0:1788575134.849258:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575134.849261:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575134.849264:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575134.849268:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849272:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575134.849274:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849276:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575134.849280:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575134.849280:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575134.849281:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575134.849284:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575134.849288:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575134.849291:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575134.849296:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849298:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575134.849301:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849303:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575134.849307:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575134.849309:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849314:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575134.849317:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575134.849320:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575134.849322:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849325:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575134.849327:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575134.849330:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849332:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575134.849340:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575134.849342:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575134.849344:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575134.849345:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849346:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575134.849347:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575134.849348:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575134.849352:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575134.849354:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575134.849361:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516784768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575134.849371:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575134.849375:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575134.849382:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516784768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575134.849393:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575134.849396:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575134.849399:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575134.849403:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575134.849406:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575134.849410:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575134.849414:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15822 00000400:00000200:0.0:1788575134.849423:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575134.849430:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575134.849433:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575134.849437:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849439:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575134.849443:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849445:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575134.849446:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575134.849452:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575134.849455:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575134.849458:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575134.849461:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575134.849463:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575134.849468:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849472:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575134.849473:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849474:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575134.849476:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575134.849480:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849481:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575134.849482:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575134.849486:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575134.849489:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849491:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575134.849494:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575134.849499:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849500:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575134.849504:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575134.849509:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575134.849512:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575134.849514:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849516:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575134.849518:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575134.849521:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575134.849525:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575134.849529:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575134.849533:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516784896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575134.849548:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575134.849551:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575134.849561:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516784896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575134.849569:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575134.849572:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575134.849577:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575134.849578:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575134.849581:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575134.849584:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575134.849587:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15822 00000400:00000200:0.0:1788575134.849592:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575134.849597:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575134.849599:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575134.849603:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849605:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575134.849607:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849611:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575134.849615:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575134.849617:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575134.849619:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575134.849622:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575134.849626:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575134.849627:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575134.849630:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849633:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575134.849636:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849638:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575134.849641:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575134.849647:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575134.849649:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575134.849654:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575134.849657:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575134.849660:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849661:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575134.849663:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575134.849666:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849668:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575134.849672:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575134.849675:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575134.849679:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575134.849682:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575134.849684:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575134.849686:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575134.849688:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575134.849691:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575134.849696:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575134.849702:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516785024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575134.849713:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575134.849716:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575134.849723:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516785024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575134.849745:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575134.849748:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15832) 00000100:00000001:0.0:1788575135.360954:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575135.360958:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575135.360960:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575135.360962:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575135.360968:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575135.360971:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575135.360973:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575135.360976:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575135.360979:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575136.064991:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575136.065009:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575136.065011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575136.384942:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575136.384947:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575136.384953:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575136.384958:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575136.384965:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575136.384970:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575136.384975:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575136.384979:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575136.384984:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575137.408915:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575137.408920:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575137.408924:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575137.408929:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575137.408933:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575137.408938:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575137.408943:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575137.408945:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575137.408947:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575138.112983:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575138.112986:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575138.112988:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575138.688905:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310498541, at 4310498560 00000100:00000010:0.0:1788575138.688918:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfb40. 00000020:00000001:0.0:1788575139.136976:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575139.136979:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575139.136982:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575139.456966:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575139.456970:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575139.456973:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575139.456977:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575139.456980:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575139.456984:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575139.456995:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575139.456997:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575139.457000:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575139.968867:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575139.968876:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575139.968878:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575139.968882:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575139.968886:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575139.968892:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15827 00000400:00000200:0.0:1788575139.968900:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575139.968906:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575139.968909:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575139.968912:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.968915:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575139.968918:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.968920:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575139.968924:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575139.968926:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575139.968929:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575139.968932:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575139.968933:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575139.968935:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575139.968937:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.968939:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575139.968940:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.968941:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575139.968943:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575139.968946:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575139.968947:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575139.968949:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575139.968952:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575139.968954:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.968955:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575139.968957:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575139.968977:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575139.968980:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575139.968983:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575139.968984:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.968986:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575139.969000:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575139.969004:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575139.969010:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575139.969015:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575139.969022:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516785152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575139.969030:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575139.969033:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575139.969036:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516785152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575139.969052:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575139.969055:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575139.969059:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575139.969060:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575139.969061:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575139.969064:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575139.969066:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15827 00000400:00000200:0.0:1788575139.969070:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575139.969073:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575139.969075:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575139.969078:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969079:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575139.969081:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969083:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575139.969085:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575139.969086:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575139.969093:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575139.969095:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575139.969097:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575139.969099:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575139.969102:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969104:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575139.969106:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969107:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575139.969110:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575139.969112:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969114:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575139.969116:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575139.969121:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575139.969122:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.969124:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575139.969127:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575139.969130:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.969132:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575139.969135:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575139.969137:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575139.969140:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575139.969141:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.969142:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575139.969144:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575139.969145:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575139.969148:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575139.969152:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575139.969156:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516785280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575139.969162:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575139.969164:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575139.969168:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516785280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575139.969180:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575139.969183:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575139.969186:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575139.969187:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575139.969189:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575139.969191:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575139.969193:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15827 00000400:00000200:0.0:1788575139.969196:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575139.969199:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575139.969201:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575139.969204:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969206:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575139.969208:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969209:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575139.969211:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575139.969213:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575139.969214:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575139.969216:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575139.969217:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575139.969220:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575139.969222:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969224:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575139.969226:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969228:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575139.969229:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575139.969231:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969233:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575139.969237:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575139.969240:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575139.969242:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.969244:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575139.969246:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575139.969248:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.969249:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575139.969252:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575139.969254:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575139.969257:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575139.969258:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.969259:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575139.969261:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575139.969262:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575139.969265:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575139.969269:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575139.969273:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516785408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575139.969279:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575139.969281:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575139.969285:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516785408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575139.969291:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575139.969293:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575139.969296:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575139.969298:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575139.969299:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575139.969302:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575139.969304:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15827 00000400:00000200:0.0:1788575139.969308:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575139.969310:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575139.969313:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575139.969319:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969321:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575139.969323:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969325:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575139.969326:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575139.969328:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575139.969329:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575139.969331:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575139.969333:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575139.969335:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575139.969337:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969339:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575139.969340:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969342:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575139.969344:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575139.969346:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969349:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575139.969350:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575139.969353:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575139.969355:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.969357:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575139.969359:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575139.969360:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.969362:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575139.969365:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575139.969367:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575139.969369:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575139.969371:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.969372:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575139.969373:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575139.969375:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575139.969378:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575139.969384:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575139.969388:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516785536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575139.969396:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575139.969398:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575139.969401:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516785536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575139.969408:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575139.969410:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575139.969413:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575139.969415:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575139.969416:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575139.969419:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575139.969421:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15827 00000400:00000200:0.0:1788575139.969424:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575139.969427:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575139.969429:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575139.969432:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969434:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575139.969436:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969438:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575139.969440:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575139.969441:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575139.969442:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575139.969444:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575139.969445:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575139.969447:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575139.969450:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969452:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575139.969459:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969461:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575139.969463:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575139.969466:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575139.969468:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575139.969470:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575139.969472:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575139.969474:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.969476:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575139.969477:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575139.969479:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.969480:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575139.969489:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575139.969491:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575139.969493:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575139.969494:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575139.969496:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575139.969498:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575139.969499:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575139.969504:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575139.969507:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575139.969511:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516785664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575139.969518:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575139.969519:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575139.969522:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516785664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575139.969530:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575139.969534:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15837) 00000100:00000001:0.0:1788575140.480909:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575140.480915:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575140.480917:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575140.480928:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575140.480935:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575140.480940:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575140.480953:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575140.480957:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575140.480970:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575141.184987:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575141.184990:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575141.184993:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575141.504921:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575141.504927:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575141.504931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575141.504936:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575141.504941:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575141.504946:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575141.504954:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575141.504957:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575141.504964:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575142.528942:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575142.528947:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575142.528951:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575142.528957:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575142.528963:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575142.528967:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575142.528976:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575142.528978:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575142.528981:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575143.232997:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575143.233000:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575143.233002:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575143.552932:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575143.552938:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575143.552941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575143.552947:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575143.552953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575143.552959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575143.552967:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575143.552971:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575143.552992:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575143.808877:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310503661, at 4310503680 00000100:00000010:0.0:1788575143.808893:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf2a0. 00000020:00000001:0.0:1788575144.256930:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575144.256932:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575144.256935:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575144.576915:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575144.576921:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575144.576923:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575144.576930:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575144.576938:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575144.576950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575144.576956:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575144.576960:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575144.576966:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575145.600912:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575145.600920:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575145.600924:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575145.600932:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575145.600946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575145.600955:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575145.600968:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575145.600976:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575145.600987:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575145.601046:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575145.601049:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575145.601051:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575145.601053:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575145.601056:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575145.601061:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575145.601063:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575145.601065:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575145.601068:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575146.304977:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575146.304981:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575146.304984:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575146.624978:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575146.625000:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575146.625004:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575146.625008:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575146.625022:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575146.625027:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575146.625050:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575146.625054:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575146.625058:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575147.648886:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575147.648891:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575147.648894:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575147.648901:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575147.648907:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575147.648913:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575147.648931:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575147.648934:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575147.648937:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575148.353010:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575148.353012:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575148.353021:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575148.672932:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575148.672939:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575148.672942:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575148.672948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575148.672953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575148.672959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575148.672977:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575148.672987:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575148.672993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575148.928954:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310508781, at 4310508800 00000100:00000010:0.0:1788575148.928976:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf120. 00000020:00000001:0.0:1788575149.377025:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575149.377029:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575149.377031:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575149.696929:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575149.696937:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575149.696941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575149.696967:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575149.696976:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575149.696984:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575149.697005:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575149.697016:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575149.697022:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575150.720883:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575150.720890:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575150.720892:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575150.720897:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575150.720901:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575150.720908:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575150.720914:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575150.720929:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575150.720937:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575150.720969:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575150.720973:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575150.720977:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575150.720982:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575150.720985:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575150.720988:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575150.720999:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575150.721002:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575150.721005:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575151.424989:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575151.424991:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575151.424994:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575151.744917:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575151.744925:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575151.744929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575151.744936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575151.744941:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575151.744945:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575151.744950:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575151.744953:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575151.744958:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575151.744991:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575151.744995:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575151.745003:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575151.745006:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575151.745009:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575151.745011:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575151.745013:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575151.745014:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575151.745016:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575152.768895:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575152.768902:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575152.768904:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575152.768910:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575152.768916:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575152.768921:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575152.768927:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575152.768931:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575152.768934:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575152.768969:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575152.768972:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575152.768974:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575152.768976:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575152.768979:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575152.768982:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575152.768987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575152.768989:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575152.768995:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575153.472969:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575153.472974:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575153.472978:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575153.792931:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575153.792938:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575153.792945:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575153.792953:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575153.792962:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575153.792969:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575153.792976:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575153.792979:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575153.792984:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575153.793012:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575153.793032:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575153.793033:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575153.793036:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575153.793038:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575153.793039:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575153.793041:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575153.793042:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575153.793043:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575154.048920:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310513902, at 4310513920 00000100:00000010:0.0:1788575154.048931:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf780. 00000020:00000001:0.0:1788575154.496994:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575154.496998:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575154.497002:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575154.816983:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575154.816988:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575154.816991:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575154.816996:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575154.817001:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575154.817006:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575154.817010:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575154.817012:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575154.817014:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575154.817036:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575154.817038:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575154.817039:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575154.817041:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575154.817046:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575154.817052:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575154.817055:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575154.817057:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575154.817061:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575154.817087:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575154.817104:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575154.817105:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575154.817107:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575154.817109:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575154.817111:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575154.817130:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575154.817133:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575154.817138:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575155.840951:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575155.840960:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575155.840963:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575155.840972:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575155.840979:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575155.840988:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575155.841004:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575155.841008:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575155.841012:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575156.544978:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575156.544981:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575156.544984:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575156.864963:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575156.864981:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575156.864984:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575156.864991:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575156.864999:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575156.865005:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575156.865009:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575156.865014:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575156.865023:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575156.865086:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575156.865100:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575156.865109:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575156.865120:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575156.865131:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575156.865139:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575156.865145:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575156.865147:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575156.865151:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575157.888953:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575157.888963:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575157.888967:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575157.888975:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575157.888983:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575157.889014:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575157.889018:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575157.889021:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575157.889023:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575158.593111:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575158.593116:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575158.593119:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575158.912922:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575158.912927:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575158.912932:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575158.912937:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575158.912943:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575158.912950:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575158.912954:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575158.912959:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575158.912963:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575159.168926:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310519021, at 4310519040 00000100:00000010:0.0:1788575159.168945:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfc60. 00000020:00000001:0.0:1788575159.616967:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575159.616971:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575159.616974:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575160.448942:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575160.448956:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575160.448959:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575160.448963:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575160.448968:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575160.448977:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15847 00000400:00000200:0.0:1788575160.448988:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575160.448996:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575160.448999:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575160.449002:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449005:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575160.449007:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449009:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575160.449037:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575160.449039:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575160.449042:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575160.449045:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575160.449048:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575160.449050:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575160.449053:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449055:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575160.449056:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449057:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575160.449060:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575160.449062:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449064:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575160.449067:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575160.449070:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575160.449073:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.449075:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575160.449077:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575160.449086:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575160.449089:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575160.449093:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575160.449094:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.449096:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575160.449099:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575160.449104:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575160.449113:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575160.449121:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575160.449139:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516787712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575160.449149:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575160.449153:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575160.449157:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516787712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575160.449180:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575160.449183:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575160.449186:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575160.449187:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575160.449189:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575160.449191:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575160.449194:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15847 00000400:00000200:0.0:1788575160.449198:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575160.449201:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575160.449222:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575160.449230:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449235:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575160.449239:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449242:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575160.449245:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575160.449252:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575160.449256:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575160.449261:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575160.449266:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575160.449269:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575160.449273:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449277:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575160.449280:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449283:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575160.449286:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575160.449289:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449292:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575160.449301:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575160.449308:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575160.449313:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.449315:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575160.449322:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575160.449325:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.449327:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575160.449334:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575160.449338:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575160.449345:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575160.449348:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.449354:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575160.449358:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575160.449362:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575160.449370:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575160.449378:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575160.449392:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516787840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575160.449401:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575160.449405:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575160.449410:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516787840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575160.449423:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575160.449427:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575160.449434:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575160.449435:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575160.449437:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575160.449440:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575160.449444:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15847 00000400:00000200:0.0:1788575160.449455:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575160.449465:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575160.449472:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575160.449488:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449494:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575160.449499:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449502:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575160.449507:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575160.449508:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575160.449509:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575160.449512:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575160.449515:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575160.449519:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575160.449524:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449527:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575160.449532:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449537:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575160.449539:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575160.449545:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449549:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575160.449552:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575160.449556:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575160.449558:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.449564:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575160.449567:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575160.449570:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.449572:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575160.449575:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575160.449579:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575160.449581:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575160.449583:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.449587:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575160.449589:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575160.449592:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575160.449597:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575160.449610:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575160.449618:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516787968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575160.449637:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575160.449641:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575160.449648:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516787968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575160.449660:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575160.449665:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575160.449676:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575160.449678:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575160.449680:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575160.449682:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575160.449687:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15847 00000400:00000200:0.0:1788575160.449692:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575160.449696:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575160.449698:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575160.449704:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449709:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575160.449714:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449717:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575160.449721:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575160.449724:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575160.449728:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575160.449730:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575160.449734:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575160.449739:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575160.449744:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449760:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575160.449774:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449778:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575160.449782:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575160.449789:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575160.449794:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575160.449795:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575160.449799:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575160.449803:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.449845:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575160.449850:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575160.449852:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.449853:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575160.449858:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575160.449862:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575160.449866:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575160.449870:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.449872:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575160.449875:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575160.449878:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575160.449886:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575160.449892:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575160.449900:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516788096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575160.449923:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575160.449927:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575160.449935:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516788096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575160.449949:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575160.449953:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575160.449958:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575160.449962:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575160.449971:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575160.449978:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575160.449984:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15847 00000400:00000200:0.0:1788575160.449991:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575160.449995:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575160.449999:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575160.450006:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.450011:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575160.450015:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.450019:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575160.450022:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575160.450023:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575160.450024:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575160.450026:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575160.450029:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575160.450031:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575160.450037:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.450040:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575160.450044:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575160.450047:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575160.450052:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575160.450057:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575160.450059:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575160.450061:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575160.450065:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575160.450071:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.450074:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575160.450078:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575160.450082:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.450084:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575160.450095:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575160.450100:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575160.450106:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575160.450108:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575160.450112:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575160.450115:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575160.450118:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575160.450123:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575160.450129:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575160.450139:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516788224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575160.450155:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575160.450160:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575160.450166:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516788224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575160.450182:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575160.450187:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15858) 00000100:00000001:0.0:1788575160.960904:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575160.960912:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575160.960917:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575160.960921:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575160.960928:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575160.960932:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575160.960952:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575160.960957:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575160.960963:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575161.665017:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575161.665021:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575161.665024:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575161.984920:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575161.984930:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575161.984933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575161.984939:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575161.984945:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575161.984950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575161.984985:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575161.984988:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575161.984991:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575161.985028:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575161.985032:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575161.985036:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575161.985042:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575161.985047:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575161.985050:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575161.985061:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575161.985064:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575161.985066:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575163.009024:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575163.009039:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575163.009054:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575163.009066:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575163.009072:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575163.009079:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575163.009085:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575163.009088:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575163.009106:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575163.713017:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575163.713023:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575163.713026:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575164.032911:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575164.032920:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575164.032925:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575164.032936:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575164.032946:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575164.032952:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575164.032966:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575164.032970:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575164.032974:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575164.288956:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310524141, at 4310524160 00000100:00000010:0.0:1788575164.288975:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf9c0. 00000020:00000001:0.0:1788575164.737073:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575164.737079:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575164.737104:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575165.056951:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575165.056958:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575165.056962:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575165.056969:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575165.056975:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575165.056981:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575165.056990:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575165.056996:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575165.057006:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575166.080967:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575166.080973:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575166.080978:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575166.080984:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575166.080988:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575166.080994:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575166.081012:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575166.081014:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575166.081017:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575166.785093:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575166.785097:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575166.785100:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575167.104988:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575167.104994:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575167.105000:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575167.105010:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575167.105023:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575167.105035:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575167.105048:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575167.105054:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575167.105062:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575168.128898:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575168.128933:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575168.128937:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575168.128945:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575168.128951:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575168.128958:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575168.128973:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575168.128993:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575168.128997:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575168.833001:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575168.833005:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575168.833010:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575169.152918:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575169.152926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575169.152930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575169.152936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575169.152943:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575169.152949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575169.152961:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575169.152965:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575169.152970:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575169.408910:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310529261, at 4310529280 00000100:00000010:0.0:1788575169.408925:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf1e0. 00000020:00000001:0.0:1788575169.856997:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575169.857000:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575169.857002:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575170.176908:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575170.176915:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575170.176919:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575170.176925:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575170.176932:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575170.176938:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575170.176943:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575170.176946:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575170.176950:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575171.200950:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575171.200956:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575171.200959:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575171.200967:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575171.200973:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575171.200978:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575171.200987:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575171.200991:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575171.200996:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575171.905014:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575171.905018:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575171.905020:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575172.224935:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575172.224942:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575172.224947:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575172.224956:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575172.224964:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575172.224971:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575172.224977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575172.224984:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575172.224989:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575173.248973:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575173.248992:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575173.248996:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575173.249005:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575173.249013:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575173.249019:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575173.249031:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575173.249035:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575173.249038:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575173.953034:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575173.953037:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575173.953040:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575174.272900:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575174.272909:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575174.272924:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575174.272932:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575174.272938:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575174.272945:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575174.272961:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575174.272965:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575174.272968:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575174.528893:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310534382, at 4310534400 00000100:00000010:0.0:1788575174.528922:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfd80. 00000020:00000001:0.0:1788575174.977032:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575174.977036:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575174.977053:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575175.296957:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575175.296963:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575175.296968:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575175.296973:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575175.296978:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575175.296983:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575175.297010:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575175.297016:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575175.297024:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575175.809000:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575175.809011:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575175.809013:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575175.809017:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575175.809022:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575175.809029:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15863 00000400:00000200:0.0:1788575175.809038:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575175.809044:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575175.809047:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575175.809050:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809054:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575175.809056:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809059:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575175.809062:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575175.809065:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575175.809067:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575175.809070:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575175.809072:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575175.809075:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575175.809078:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809080:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575175.809082:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809101:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575175.809105:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575175.809109:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809112:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575175.809115:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575175.809118:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575175.809122:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.809124:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575175.809126:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575175.809133:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575175.809136:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575175.809140:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575175.809141:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.809144:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575175.809147:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575175.809151:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575175.809158:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575175.809165:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575175.809183:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516789632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575175.809194:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575175.809199:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575175.809204:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516789632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575175.809227:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575175.809229:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575175.809232:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575175.809233:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575175.809235:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575175.809237:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575175.809239:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15863 00000400:00000200:0.0:1788575175.809248:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575175.809251:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575175.809253:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575175.809255:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809257:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575175.809259:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809261:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575175.809266:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575175.809267:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575175.809269:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575175.809302:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575175.809306:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575175.809309:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575175.809315:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809321:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575175.809325:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809327:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575175.809331:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575175.809338:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809342:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575175.809343:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575175.809354:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575175.809360:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.809363:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575175.809368:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575175.809371:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.809373:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575175.809380:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575175.809382:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575175.809384:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575175.809390:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.809392:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575175.809394:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575175.809397:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575175.809402:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575175.809409:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575175.809417:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516789760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575175.809445:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575175.809448:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575175.809454:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516789760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575175.809463:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575175.809466:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575175.809468:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575175.809472:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575175.809473:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575175.809476:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575175.809480:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15863 00000400:00000200:0.0:1788575175.809485:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575175.809491:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575175.809494:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575175.809496:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809498:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575175.809502:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809515:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575175.809518:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575175.809520:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575175.809524:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575175.809527:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575175.809530:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575175.809541:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575175.809546:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809549:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575175.809554:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809556:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575175.809561:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575175.809566:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809569:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575175.809573:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575175.809576:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575175.809577:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.809579:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575175.809581:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575175.809585:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.809589:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575175.809592:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575175.809601:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575175.809606:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575175.809607:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.809608:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575175.809609:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575175.809611:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575175.809618:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575175.809623:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575175.809631:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516789888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575175.809641:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575175.809645:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575175.809654:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516789888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575175.809668:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575175.809675:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575175.809679:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575175.809681:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575175.809684:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575175.809687:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575175.809692:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15863 00000400:00000200:0.0:1788575175.809699:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575175.809702:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575175.809706:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575175.809711:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809714:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575175.809719:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809721:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575175.809724:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575175.809725:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575175.809728:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575175.809730:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575175.809734:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575175.809738:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575175.809745:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809754:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575175.809762:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809768:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575175.809773:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575175.809780:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575175.809785:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575175.809792:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575175.809796:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575175.809798:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.809804:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575175.809852:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575175.809855:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.809860:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575175.809864:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575175.809868:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575175.809873:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575175.809875:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.809880:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575175.809882:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575175.809886:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575175.809946:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575175.809954:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575175.809967:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516790016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575175.809984:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575175.809989:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575175.810000:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516790016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575175.810011:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575175.810014:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575175.810022:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575175.810024:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575175.810027:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575175.810032:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575175.810035:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15863 00000400:00000200:0.0:1788575175.810039:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575175.810045:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575175.810048:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575175.810054:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.810058:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575175.810062:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.810072:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575175.810075:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575175.810078:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575175.810079:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575175.810084:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575175.810086:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575175.810091:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575175.810095:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.810098:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575175.810102:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575175.810104:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575175.810109:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575175.810114:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575175.810117:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575175.810119:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575175.810122:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575175.810127:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.810131:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575175.810134:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575175.810139:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.810142:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575175.810146:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575175.810150:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575175.810155:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575175.810157:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575175.810163:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575175.810165:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575175.810168:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575175.810173:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575175.810177:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575175.810185:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516790144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575175.810206:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575175.810209:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575175.810215:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516790144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575175.810225:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575175.810231:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15873) 00000100:00000001:0.0:1788575176.321025:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575176.321030:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575176.321035:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575176.321040:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575176.321044:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575176.321051:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575176.321057:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575176.321061:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575176.321068:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575177.024976:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575177.024979:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575177.024981:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575177.344904:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575177.344912:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575177.344916:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575177.344921:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575177.344928:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575177.344934:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575177.344947:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575177.344951:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575177.344953:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575178.368925:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575178.368930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575178.368932:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575178.368938:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575178.368943:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575178.368947:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575178.368956:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575178.368963:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575178.368966:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575179.072960:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575179.072962:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575179.072965:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575179.392890:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575179.392896:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575179.392900:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575179.392907:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575179.392913:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575179.392917:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575179.392940:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575179.392943:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575179.392946:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575179.648888:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310539501, at 4310539520 00000100:00000010:0.0:1788575179.648903:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf4e0. 00000020:00000001:0.0:1788575180.096981:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575180.096984:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575180.096986:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575180.416864:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575180.416871:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575180.416876:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575180.416882:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575180.416890:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575180.416894:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575180.416913:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575180.416916:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575180.416920:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575181.440903:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575181.440909:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575181.440913:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575181.440917:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575181.440920:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575181.440924:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575181.440941:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575181.440944:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575181.440948:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575182.145009:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575182.145012:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575182.145014:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575182.464907:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575182.464915:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575182.464919:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575182.464925:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575182.464930:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575182.464936:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575182.464947:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575182.464949:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575182.464959:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575184.193001:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575184.193005:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575184.193007:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575184.512915:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575184.512920:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575184.512924:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575184.512929:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575184.512936:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575184.512940:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575184.512954:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575184.512957:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575184.512962:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575184.768885:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310544621, at 4310544640 00000100:00000010:0.0:1788575184.768901:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfa80. 00000020:00000001:0.0:1788575185.217007:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575185.217011:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575185.217014:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575185.536890:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575185.536896:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575185.536899:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575185.536906:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575185.536913:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575185.536918:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575185.536924:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575185.536930:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575185.536941:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575186.049072:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788575186.049080:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575186.049095:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575186.049103:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575186.049110:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575186.560869:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575186.560875:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575186.560878:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575186.560884:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575186.560891:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575186.560897:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575186.560903:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575186.560907:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575186.560912:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575187.264985:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575187.264988:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575187.264990:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575187.584952:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575187.584960:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575187.584964:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575187.584967:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575187.584971:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575187.584974:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575187.584979:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575187.584983:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575187.584988:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575188.608931:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575188.608936:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575188.608939:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575188.608949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575188.608955:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575188.608958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575188.608975:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575188.608979:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575188.608982:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575189.312985:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575189.312988:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575189.313026:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575189.632911:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575189.632915:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575189.632919:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575189.632922:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575189.632926:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575189.632929:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575189.632953:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575189.632957:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575189.632960:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575189.888888:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310549741, at 4310549760 00000100:00000010:0.0:1788575189.888905:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf8a0. 00000020:00000001:0.0:1788575190.336984:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575190.336987:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575190.336989:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575190.656947:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575190.656951:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575190.656955:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575190.656959:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575190.656962:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575190.656965:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575190.656968:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575190.656970:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575190.656972:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575191.168934:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575191.168944:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575191.168947:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575191.168951:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575191.168956:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575191.168963:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15878 00000400:00000200:0.0:1788575191.168971:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575191.168976:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575191.168979:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575191.168981:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.168999:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575191.169002:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169004:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575191.169008:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575191.169009:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575191.169012:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575191.169015:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575191.169018:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575191.169020:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575191.169023:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169025:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575191.169027:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169029:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575191.169031:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575191.169034:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169037:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575191.169039:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575191.169042:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575191.169045:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.169047:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575191.169049:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575191.169055:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575191.169058:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575191.169061:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575191.169063:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.169066:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575191.169069:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575191.169073:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575191.169080:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575191.169087:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575191.169095:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516791552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575191.169116:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575191.169120:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575191.169125:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516791552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575191.169145:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575191.169148:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575191.169150:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575191.169152:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575191.169153:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575191.169156:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575191.169160:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15878 00000400:00000200:0.0:1788575191.169164:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575191.169169:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575191.169171:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575191.169177:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169181:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575191.169185:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169189:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575191.169192:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575191.169195:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575191.169198:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575191.169202:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575191.169207:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575191.169212:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575191.169218:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169221:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575191.169226:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169230:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575191.169240:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575191.169247:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169252:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575191.169258:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575191.169267:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575191.169271:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.169276:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575191.169283:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575191.169287:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.169290:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575191.169297:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575191.169301:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575191.169306:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575191.169310:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.169314:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575191.169316:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575191.169320:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575191.169328:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575191.169334:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575191.169346:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516791680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575191.169361:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575191.169366:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575191.169374:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516791680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575191.169397:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575191.169401:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575191.169409:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575191.169411:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575191.169415:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575191.169422:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575191.169434:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15878 00000400:00000200:0.0:1788575191.169444:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575191.169450:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575191.169455:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575191.169460:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169462:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575191.169465:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169467:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575191.169470:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575191.169474:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575191.169476:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575191.169479:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575191.169484:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575191.169489:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575191.169496:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169500:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575191.169505:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169512:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575191.169517:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575191.169525:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169531:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575191.169534:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575191.169537:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575191.169542:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.169547:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575191.169550:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575191.169555:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.169559:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575191.169565:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575191.169570:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575191.169576:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575191.169584:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.169588:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575191.169591:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575191.169595:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575191.169602:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575191.169607:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575191.169616:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516791808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575191.169626:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575191.169631:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575191.169638:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516791808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575191.169655:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575191.169660:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575191.169669:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575191.169673:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575191.169676:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575191.169681:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575191.169686:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15878 00000400:00000200:0.0:1788575191.169694:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575191.169700:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575191.169704:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575191.169712:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169717:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575191.169722:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169727:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575191.169732:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575191.169735:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575191.169738:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575191.169741:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575191.169745:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575191.169758:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575191.169766:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169771:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575191.169777:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169782:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575191.169788:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575191.169795:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575191.169801:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575191.169806:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575191.169863:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575191.169870:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.169875:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575191.169880:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575191.169885:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.169889:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575191.169907:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575191.169913:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575191.169919:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575191.169922:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.169928:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575191.169930:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575191.169935:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575191.169943:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575191.169950:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575191.169959:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516791936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575191.169977:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575191.169982:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575191.169992:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516791936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575191.170014:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575191.170029:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575191.170036:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575191.170038:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575191.170039:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575191.170046:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575191.170053:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15878 00000400:00000200:0.0:1788575191.170065:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575191.170074:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575191.170080:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575191.170089:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.170096:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575191.170102:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.170109:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575191.170115:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575191.170119:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575191.170124:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575191.170129:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575191.170135:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575191.170141:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575191.170151:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.170157:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575191.170164:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575191.170170:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575191.170175:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575191.170182:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575191.170188:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575191.170193:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575191.170199:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575191.170205:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.170211:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575191.170223:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575191.170227:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.170231:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575191.170239:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575191.170244:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575191.170250:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575191.170253:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.170257:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575191.170260:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575191.170265:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575191.170272:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575191.170279:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575191.170289:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516792064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575191.170303:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575191.170308:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575191.170318:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516792064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575191.170341:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575191.170348:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15888) 00000100:00000001:0.0:1788575191.680888:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575191.680897:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575191.680909:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575191.680915:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575191.680921:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575191.680926:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575191.680939:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575191.680943:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575191.680947:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575191.680975:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575191.680979:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575191.680981:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575191.680984:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575191.680987:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575191.680996:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575191.680998:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575191.680999:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575191.681001:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575192.384973:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575192.384976:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575192.384978:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575192.704953:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575192.704957:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575192.704958:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575192.704961:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575192.704963:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575192.704965:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575192.704967:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575192.704968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575192.704970:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575193.728952:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575193.728961:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575193.728966:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575193.728974:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575193.728982:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575193.728986:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575193.728991:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575193.728994:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575193.729001:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575194.432987:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575194.432991:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575194.432993:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575194.752978:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575194.752988:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575194.752999:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575194.753014:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575194.753031:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575194.753036:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575194.753044:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575194.753046:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575194.753055:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575195.008955:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310554861, at 4310554880 00000100:00000010:0.0:1788575195.008988:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfd20. 00000020:00000001:0.0:1788575195.456973:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575195.456977:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575195.456981:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575195.776931:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575195.776937:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575195.776947:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575195.776956:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575195.776969:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575195.776983:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575195.776994:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575195.776998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575195.777002:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575195.777050:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575195.777057:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575195.777060:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575195.777063:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575195.777066:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575195.777067:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575195.777069:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575195.777070:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575195.777071:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575196.800912:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575196.800919:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575196.800923:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575196.800929:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575196.800934:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575196.800941:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575196.800961:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575196.800965:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575196.800967:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575197.505043:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575197.505049:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575197.505052:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575197.824909:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575197.824918:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575197.824922:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575197.824943:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575197.824952:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575197.824961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575197.824972:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575197.824976:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575197.824985:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575197.825052:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575197.825056:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575197.825057:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575197.825059:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575197.825061:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575197.825063:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575197.825065:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575197.825066:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575197.825068:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575198.848971:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575198.848977:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575198.848983:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575198.848997:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575198.849004:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575198.849009:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575198.849014:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575198.849018:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575198.849021:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575199.552982:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575199.552985:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575199.552987:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575199.872982:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575199.872991:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575199.872996:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575199.873007:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575199.873014:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575199.873019:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575199.873024:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575199.873026:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575199.873029:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575200.128898:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310559982, at 4310560000 00000100:00000010:0.0:1788575200.128917:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf600. 00000020:00000001:0.0:1788575200.576944:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575200.576947:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575200.576949:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575200.896974:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575200.896981:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575200.896986:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575200.897000:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575200.897009:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575200.897017:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575200.897039:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575200.897046:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575200.897061:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575201.920947:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575201.920954:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575201.920964:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575201.920972:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575201.920988:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575201.920992:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575201.921001:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575201.921005:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575201.921010:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575202.624982:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575202.624987:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575202.624991:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575202.944924:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575202.944930:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575202.944932:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575202.944937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575202.944941:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575202.944946:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575202.944950:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575202.944953:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575202.944962:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575202.944999:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575202.945003:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575202.945004:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575202.945007:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575202.945010:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575202.945024:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575202.945033:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575202.945037:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575202.945042:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575203.968932:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575203.968939:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575203.968941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575203.968947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575203.968953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575203.968958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575203.968968:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575203.968973:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575203.968984:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575203.969028:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575203.969035:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575203.969039:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575203.969046:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575203.969055:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575203.969059:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575203.969070:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575203.969073:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575203.969080:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575204.673037:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575204.673041:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575204.673044:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575204.992910:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575204.992917:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575204.992919:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575204.992924:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575204.992930:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575204.992934:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575204.992936:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575204.992938:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575204.992941:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575205.248913:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310565101, at 4310565120 00000100:00000010:0.0:1788575205.248925:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfde0. 00000020:00000001:0.0:1788575205.697015:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575205.697034:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575205.697038:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575206.016908:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575206.016916:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575206.016918:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575206.016926:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575206.016931:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575206.016935:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575206.016958:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575206.016963:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575206.016966:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575207.040925:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575207.040930:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575207.040935:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575207.040939:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575207.040942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575207.040944:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575207.040947:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575207.040949:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575207.040951:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575207.041100:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575207.041102:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575207.041104:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575207.041106:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575207.041108:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575207.041110:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575207.041112:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575207.041114:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575207.041115:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575207.744948:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575207.744952:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575207.744954:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575208.064907:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575208.064911:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575208.064914:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575208.064920:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575208.064924:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575208.064928:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575208.064947:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575208.064949:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575208.064952:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575209.088898:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575209.088902:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575209.088913:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575209.088920:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575209.088927:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575209.088932:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575209.088946:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575209.088949:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575209.088953:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575209.089001:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575209.089004:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575209.089005:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575209.089007:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575209.089009:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575209.089010:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575209.089012:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575209.089013:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575209.089014:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575209.792985:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575209.792989:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575209.792992:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575210.112899:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575210.112904:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575210.112908:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575210.112914:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575210.112919:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575210.112923:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575210.112948:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575210.112951:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575210.112954:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575210.368913:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310570221, at 4310570240 00000100:00000010:0.0:1788575210.368939:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf6c0. 00000020:00000001:0.0:1788575210.817003:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575210.817006:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575210.817022:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575211.136947:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575211.136951:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575211.136954:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575211.136959:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575211.136963:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575211.136966:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575211.136977:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575211.136980:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575211.136983:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575212.160913:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575212.160918:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575212.160922:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575212.160927:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575212.160933:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575212.160939:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575212.160957:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575212.160960:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575212.160964:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575212.865023:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575212.865026:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575212.865028:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575213.184976:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575213.184987:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575213.184990:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575213.184993:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575213.184996:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575213.184999:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575213.185003:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575213.185006:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575213.185010:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575214.208899:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575214.208904:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575214.208908:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575214.208915:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575214.208922:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575214.208928:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575214.208950:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575214.208958:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575214.208962:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575214.912974:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575214.912977:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575214.912979:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575215.232923:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575215.232929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575215.232933:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575215.232939:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575215.232944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575215.232949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575215.232959:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575215.232965:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575215.232968:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575215.488879:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310575341, at 4310575360 00000100:00000010:0.0:1788575215.488892:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf5a0. 00000020:00000001:0.0:1788575215.936981:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575215.936986:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575215.936988:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575217.280909:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575217.280915:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575217.280919:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575217.280923:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575217.280929:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575217.280934:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575217.280937:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575217.280941:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575217.280948:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575217.985025:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575217.985028:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575217.985031:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575218.304940:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575218.304947:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575218.304952:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575218.304960:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575218.304966:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575218.304971:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575218.304990:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575218.304991:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575218.304994:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575219.328911:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575219.328920:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575219.328927:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575219.328936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575219.328942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575219.328947:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575219.328954:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575219.328958:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575219.328964:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575220.033012:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575220.033015:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575220.033018:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575220.352909:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575220.352916:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575220.352920:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575220.352926:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575220.352933:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575220.352937:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575220.352956:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575220.352960:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575220.352964:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575220.608919:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310580461, at 4310580480 00000100:00000010:0.0:1788575220.608937:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf780. 00000020:00000001:0.0:1788575221.056993:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575221.056997:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575221.057000:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575221.376980:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575221.376987:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575221.376991:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575221.376998:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575221.377003:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575221.377008:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575221.377035:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575221.377051:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575221.377083:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575222.400918:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575222.400928:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575222.400932:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575222.400941:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575222.400948:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575222.400954:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575222.400969:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575222.400976:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575222.400988:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575223.105178:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575223.105184:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575223.105187:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575223.424997:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575223.425004:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575223.425009:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575223.425016:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575223.425030:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575223.425035:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575223.425042:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575223.425046:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575223.425049:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575224.448905:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575224.448915:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575224.448920:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575224.448929:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575224.448938:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575224.448945:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575224.448951:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575224.448956:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575224.448962:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575224.448996:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575224.449002:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575224.449005:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575224.449009:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575224.449016:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575224.449019:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575224.449022:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575224.449033:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575224.449037:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575224.961209:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788575224.961216:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575224.961236:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575224.961241:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575224.961243:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788575225.153075:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575225.153081:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575225.153085:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575225.472959:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575225.472969:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575225.472972:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575225.472984:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575225.472992:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575225.473000:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575225.473006:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575225.473009:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575225.473012:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575225.473030:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575225.473034:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575225.473038:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575225.473040:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575225.473045:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575225.473048:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575225.473052:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575225.473053:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575225.473055:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575225.728900:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310585582, at 4310585600 00000100:00000010:0.0:1788575225.728919:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf3c0. 00000020:00000001:0.0:1788575226.177134:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575226.177140:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575226.177144:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575227.520947:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575227.520953:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575227.520956:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575227.520969:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575227.520982:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575227.520987:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575227.520995:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575227.520998:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575227.521001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575228.225013:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575228.225017:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575228.225019:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575228.544924:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575228.544930:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575228.544934:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575228.544940:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575228.544946:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575228.544951:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575228.544969:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575228.544974:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575228.544977:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575229.568897:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575229.568904:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575229.568909:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575229.568917:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575229.568924:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575229.568931:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575229.568938:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575229.568941:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575229.568946:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575229.568973:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575229.568976:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575229.568979:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575229.568985:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575229.568989:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575229.568992:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575229.568994:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575229.568996:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575229.568998:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575230.273007:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575230.273010:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575230.273013:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575230.592915:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575230.592921:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575230.592925:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575230.592929:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575230.592932:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575230.592937:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575230.592946:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575230.592950:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575230.592953:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575230.848894:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310590701, at 4310590720 00000100:00000010:0.0:1788575230.848915:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf2a0. 00000020:00000001:0.0:1788575231.297024:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575231.297029:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575231.297032:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575231.616920:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575231.616929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575231.616935:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575231.616943:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575231.616950:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575231.616956:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575231.616972:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575231.616977:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575231.616981:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575232.640886:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575232.640892:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575232.640896:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575232.640904:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575232.640909:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575232.640915:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575232.640930:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575232.640934:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575232.640937:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575232.641060:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575232.641065:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575232.641068:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575232.641073:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575232.641079:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575232.641088:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575232.641091:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575232.641093:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575232.641095:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575233.345029:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575233.345034:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575233.345037:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575233.664884:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575233.664892:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575233.664894:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575233.664902:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575233.664911:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575233.664915:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575233.664921:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575233.664926:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575233.664935:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575233.664952:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575233.664967:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575233.664968:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575233.664971:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575233.664974:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575233.664977:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575233.664979:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575233.664982:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575233.664984:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575234.688910:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575234.688931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575234.688935:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575234.688940:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575234.688949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575234.688954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575234.688972:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575234.688977:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575234.688981:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575235.393029:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575235.393033:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575235.393035:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575235.712936:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575235.712959:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575235.712964:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575235.712972:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575235.712978:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575235.712984:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575235.712999:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575235.713002:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575235.713005:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575235.968897:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310595821, at 4310595840 00000100:00000010:0.0:1788575235.968917:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cff00. 00000020:00000001:0.0:1788575236.416993:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575236.416997:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575236.416999:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575236.736902:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575236.736918:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575236.736923:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575236.736928:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575236.736933:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575236.736937:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575236.736943:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575236.736948:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575236.736958:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575236.737004:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575236.737010:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575236.737014:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575236.737020:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575236.737028:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575236.737036:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575236.737039:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575236.737041:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575236.737043:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575237.760922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575237.760931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575237.760936:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575237.760944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575237.760951:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575237.760956:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575237.760990:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575237.760993:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575237.760996:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575238.465012:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575238.465017:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575238.465021:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575238.784884:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575238.784891:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575238.784894:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575238.784901:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575238.784909:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575238.784915:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575238.784933:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575238.784936:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575238.784941:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575238.784975:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575238.784978:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575238.784980:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575238.784982:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575238.784984:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575238.784987:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575238.784988:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575238.784990:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575238.784991:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.808946:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575239.808958:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575239.808963:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575239.808972:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575239.808985:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575239.808992:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575239.809000:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575239.809009:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575239.809013:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575239.809031:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575239.809036:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575239.809043:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575239.809051:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575239.809059:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575239.809095:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575239.809098:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575239.809101:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575239.809103:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575240.513077:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575240.513081:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575240.513084:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575240.832907:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575240.832916:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575240.832920:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575240.832930:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575240.832938:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575240.832945:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575240.832962:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575240.832972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575240.832981:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575240.833022:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575240.833026:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575240.833027:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575240.833030:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575240.833033:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575240.833035:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575240.833039:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575240.833046:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575240.833050:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575241.088920:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310600942, at 4310600960 00000100:00000010:0.0:1788575241.088939:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf180. 00000020:00000001:0.0:1788575241.536988:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575241.536992:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575241.536996:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575241.856951:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575241.856960:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575241.856970:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575241.856979:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575241.856986:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575241.857001:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575241.857014:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575241.857035:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575241.857047:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575241.857096:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575241.857100:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575241.857103:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575241.857105:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575241.857110:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575241.857113:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575241.857115:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575241.857116:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575241.857118:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.369211:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575242.369217:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575242.369222:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575242.369234:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516797952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.369254:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575242.369256:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575242.369260:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575242.369265:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575242.369269:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575242.369275:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516797952:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575242.369283:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575242.369290:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575242.369294:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575242.369300:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575242.369307:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575242.369311:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575242.369320:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575242.369326:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575242.369347:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575242.369359:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516797952, portal 25 00000100:00000001:0.0:1788575242.369363:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575242.369366:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575242.369375:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516797952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.369403:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575242.369412:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575242.369416:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575242.369423:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516797952, offset 0 00000400:00000200:0.0:1788575242.369432:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575242.369446:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575242.369456:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575242.369463:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575242.369470:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575242.369486:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575242.369500:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575242.369510:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547886:547886:256:4294967295] 192.168.202.128@tcp LPNI seq info [547809:547809:8:4294967295] 00000400:00000200:0.0:1788575242.369517:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575242.369543:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575242.369559:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575242.369568:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575242.369583:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575242.369586:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575242.369624:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575242.369630:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575242.369640:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575242.369651:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575242.369659:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575242.369664:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575242.369673:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575242.369682:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516797952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.369704:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575242.369717:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516797952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.369759:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.369765:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575242.369777:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575242.369788:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575242.369792:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310606061 00000100:00000010:0.0:1788575242.369799:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575242.369834:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575242.369843:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310606062 00000100:00000010:0.0:1788575242.369852:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50720. 00000100:00000001:0.0:1788575242.369859:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575242.369866:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575242.369871:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.369875:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575242.369883:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575242/real 1788575242] req@ffff8fc4901a5880 x1875440516797952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575242.369897:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575242.369901:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575242.369914:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575242.369923:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575242.369930:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575242.369938:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575242.369948:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516797952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575242.369967:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516797952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.369981:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575242.369987:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.369991:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575242.369995:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575242.370009:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516797952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370037:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575242.370050:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516797952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370074:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575242.370082:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370088:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575242.370093:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370098:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575242.370101:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575242.370105:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575242.370111:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575242.370114:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575242.370120:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516797952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575242.370128:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516797952:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575242.370134:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370136:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575242.370140:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516797952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370146:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575242.370149:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575242.370152:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575242.370157:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575242.370159:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575242.370160:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575242.370162:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575242.370165:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575242.370169:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575242.370175:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575242.370177:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575242.370182:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575242.370183:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370185:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370192:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575242.370194:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.370195:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575242.370196:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575242.370198:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575242.370201:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516798080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370206:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575242.370207:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575242.370209:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575242.370211:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575242.370212:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575242.370214:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798080:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575242.370216:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575242.370217:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575242.370219:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575242.370220:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575242.370222:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575242.370224:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575242.370227:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575242.370229:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575242.370232:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575242.370234:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516798080, portal 10 00000100:00000001:0.0:1788575242.370236:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575242.370237:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575242.370242:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516798080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370247:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575242.370249:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575242.370254:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575242.370256:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516798080, offset 0 00000400:00000200:0.0:1788575242.370259:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575242.370262:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575242.370265:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575242.370268:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575242.370270:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575242.370271:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575242.370273:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575242.370275:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547887:547887:256:4294967295] 192.168.202.128@tcp LPNI seq info [547810:547810:8:4294967295] 00000400:00000200:0.0:1788575242.370278:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575242.370283:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575242.370286:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575242.370289:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575242.370291:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575242.370294:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575242.370301:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575242.370303:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575242.370305:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575242.370307:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575242.370310:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575242.370311:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575242.370313:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575242.370316:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516798080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370322:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575242.370324:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516798080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370328:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.370329:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575242.370337:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575242.370341:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575242.370344:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575242.370345:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310602259 00000100:00000001:0.0:1788575242.370347:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310606061 00000100:00000001:0.0:1788575242.370349:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310602259 00000100:00000001:0.0:1788575242.370351:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310606062 00000100:00000001:0.0:1788575242.370352:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575242.370353:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.370355:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575242.370357:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575242/real 1788575242] req@ffff8fc4901a4700 x1875440516798080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575242.370363:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575242.370364:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575242.370366:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575242.370367:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575242.370368:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575242.370370:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575242.370371:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516798080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575242.370377:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516798080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370382:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575242.370383:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370384:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575242.370385:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575242.370387:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516798080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370391:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575242.370393:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516798080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370401:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575242.370403:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370404:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575242.370405:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370406:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575242.370408:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575242.370410:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575242.370412:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575242.370413:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575242.370416:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516798080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575242.370422:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798080:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575242.370424:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575242.370426:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516798208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370430:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575242.370431:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575242.370432:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575242.370433:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575242.370434:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575242.370436:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798208:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575242.370438:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575242.370439:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575242.370440:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575242.370441:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575242.370444:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575242.370445:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575242.370452:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575242.370455:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575242.370459:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575242.370462:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516798208, portal 10 00000100:00000001:0.0:1788575242.370465:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575242.370466:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575242.370474:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516798208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370480:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575242.370482:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575242.370485:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575242.370487:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516798208, offset 0 00000400:00000200:0.0:1788575242.370491:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575242.370496:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575242.370500:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575242.370502:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575242.370505:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575242.370507:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575242.370510:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575242.370512:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547888:547888:256:4294967295] 192.168.202.128@tcp LPNI seq info [547811:547811:8:4294967295] 00000400:00000200:0.0:1788575242.370517:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575242.370523:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575242.370528:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575242.370532:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575242.370536:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575242.370539:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575242.370555:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575242.370557:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575242.370560:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575242.370564:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575242.370568:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575242.370573:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575242.370577:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575242.370581:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516798208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370589:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575242.370592:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516798208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370612:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.370614:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575242.370618:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575242.370622:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575242.370625:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575242.370627:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310602266 00000100:00000001:0.0:1788575242.370630:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310606061 00000100:00000001:0.0:1788575242.370632:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310602266 00000100:00000001:0.0:1788575242.370634:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310606062 00000100:00000001:0.0:1788575242.370636:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575242.370638:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.370640:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575242.370644:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575242/real 1788575242] req@ffff8fc4901a7480 x1875440516798208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575242.370652:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575242.370654:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575242.370655:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575242.370659:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575242.370661:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575242.370663:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575242.370666:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516798208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575242.370675:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516798208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370686:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575242.370687:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370688:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575242.370690:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575242.370692:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516798208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370697:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575242.370700:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516798208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370704:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575242.370706:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370707:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575242.370708:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370709:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575242.370711:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575242.370713:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575242.370714:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575242.370716:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575242.370718:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516798208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575242.370724:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798208:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575242.370726:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575242.370729:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516798336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370736:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575242.370738:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575242.370745:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575242.370747:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575242.370749:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575242.370753:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798336:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575242.370757:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575242.370759:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575242.370761:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575242.370762:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575242.370766:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575242.370768:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575242.370773:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575242.370776:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575242.370780:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575242.370783:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516798336, portal 4 00000100:00000001:0.0:1788575242.370785:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575242.370787:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575242.370792:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516798336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370797:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575242.370799:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575242.370800:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575242.370802:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516798336, offset 0 00000400:00000200:0.0:1788575242.370805:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575242.370824:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575242.370827:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575242.370830:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575242.370832:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575242.370834:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575242.370835:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575242.370837:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547889:547889:256:4294967295] 192.168.202.128@tcp LPNI seq info [547812:547812:8:4294967295] 00000400:00000200:0.0:1788575242.370840:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575242.370844:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575242.370854:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575242.370857:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575242.370860:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575242.370862:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575242.370870:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575242.370872:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575242.370874:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575242.370876:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575242.370879:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575242.370880:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575242.370882:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575242.370885:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516798336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370891:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575242.370893:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516798336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370898:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.370899:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575242.370901:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575242.370904:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575242.370906:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575242.370908:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310602407 00000100:00000001:0.0:1788575242.370910:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310606061 00000100:00000001:0.0:1788575242.370912:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310602407 00000100:00000001:0.0:1788575242.370915:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310606062 00000100:00000001:0.0:1788575242.370917:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575242.370920:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.370922:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575242.370926:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575242/real 1788575242] req@ffff8fc4901a6300 x1875440516798336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575242.370940:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575242.370942:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575242.370944:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575242.370947:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575242.370948:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575242.370951:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575242.370956:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516798336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575242.370965:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516798336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370970:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575242.370972:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370973:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575242.370974:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575242.370977:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516798336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370981:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575242.370984:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516798336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.370988:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575242.370989:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370990:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575242.370991:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.370992:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575242.370994:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575242.370995:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575242.370998:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575242.371000:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575242.371010:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516798336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575242.371019:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798336:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575242.371023:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575242.371026:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516798464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.371034:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575242.371036:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575242.371039:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575242.371041:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575242.371043:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575242.371046:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798464:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575242.371050:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575242.371052:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575242.371053:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575242.371054:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575242.371058:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575242.371060:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575242.371064:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575242.371067:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575242.371070:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575242.371074:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516798464, portal 4 00000100:00000001:0.0:1788575242.371076:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575242.371077:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575242.371084:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516798464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.371092:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575242.371095:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575242.371097:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575242.371100:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516798464, offset 0 00000400:00000200:0.0:1788575242.371110:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575242.371113:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575242.371115:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575242.371117:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575242.371119:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575242.371121:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575242.371123:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575242.371125:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547890:547890:256:4294967295] 192.168.202.128@tcp LPNI seq info [547813:547813:8:4294967295] 00000400:00000200:0.0:1788575242.371127:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575242.371132:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575242.371135:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575242.371138:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575242.371141:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575242.371143:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575242.371150:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575242.371152:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575242.371154:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575242.371156:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575242.371159:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575242.371160:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575242.371162:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575242.371165:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516798464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.371170:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575242.371172:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516798464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.371177:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.371178:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575242.371180:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575242.371182:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575242.371184:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575242.371189:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310602409 00000100:00000001:0.0:1788575242.371191:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310606061 00000100:00000001:0.0:1788575242.371193:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310602409 00000100:00000001:0.0:1788575242.371194:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310606062 00000100:00000001:0.0:1788575242.371196:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575242.371198:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.371199:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575242.371202:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575242/real 1788575242] req@ffff8fc4901a4e00 x1875440516798464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575242.371207:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575242.371208:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575242.371209:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575242.371211:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575242.371212:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575242.371214:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575242.371216:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516798464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575242.371221:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516798464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.371225:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575242.371226:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.371227:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575242.371228:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575242.371230:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516798464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.371234:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575242.371237:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516798464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.371241:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575242.371246:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.371247:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575242.371248:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.371249:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575242.371251:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575242.371253:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575242.371255:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575242.371256:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575242.371258:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516798464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575242.371264:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798464:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575242.371267:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.371268:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575242.371270:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516798080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.371274:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575242.371275:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575242.371276:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575242.371278:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575242.371279:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575242.371279:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575242.371281:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575242.371282:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575242.371284:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575242.371285:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575242.371286:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575242.371290:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575242.371291:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.371292:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575242.371294:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516798208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.371306:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575242.371307:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575242.371308:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575242.371309:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575242.371310:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575242.371311:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575242.371312:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575242.371313:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575242.371314:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575242.371315:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575242.371316:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575242.371318:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575242.371319:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.371320:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575242.371322:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516798336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.371326:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575242.371327:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575242.371327:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575242.371329:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575242.371330:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575242.371331:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575242.371333:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575242.371334:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575242.371337:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575242.371338:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575242.371340:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575242.371344:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575242.371346:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.371347:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575242.371352:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516798464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575242.371358:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575242.371360:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575242.371368:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575242.371372:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575242.371374:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575242.371376:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575242.371378:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575242.371379:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575242.371382:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575242.371383:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575242.371384:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575242.371388:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575242.371389:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.371391:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.371396:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575242.371397:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.371399:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575242.371401:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.880927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575242.880935:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575242.880941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575242.880947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575242.880953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575242.880957:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.880961:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575242.880966:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.880981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575242.881016:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575242.881020:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575242.881027:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575242.881032:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575242.881044:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575242.881048:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.881067:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575242.881070:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575242.881074:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575243.584954:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575243.584958:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575243.584963:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575243.904920:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575243.904942:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575243.904945:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575243.904949:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575243.904957:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575243.904961:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575243.904973:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575243.904979:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575243.904995:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575243.905043:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575243.905046:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575243.905047:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575243.905049:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575243.905051:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575243.905053:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575243.905055:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575243.905058:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575243.905061:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.928965:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575244.928974:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575244.928981:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575244.928991:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575244.929003:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575244.929008:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575244.929014:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575244.929020:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575244.929026:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575244.929041:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575244.929044:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575244.929046:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575244.929048:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575244.929050:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575244.929052:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575244.929080:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575244.929083:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575244.929087:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575245.632945:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575245.632948:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575245.632951:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575245.952896:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575245.952902:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575245.952906:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575245.952911:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575245.952916:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575245.952921:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575245.952924:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575245.952926:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575245.952928:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575246.208908:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310606061, at 4310606080 00000100:00000010:0.0:1788575246.208921:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575246.657034:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575246.657038:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575246.657040:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575246.976916:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575246.976924:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575246.976929:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575246.976936:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575246.976943:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575246.976949:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575246.976971:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575246.976975:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575246.976980:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.489173:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575247.489178:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575247.489182:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575247.489191:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516798592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.489208:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575247.489211:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575247.489216:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575247.489220:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575247.489226:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575247.489236:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516798592:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575247.489245:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575247.489259:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575247.489263:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575247.489269:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575247.489275:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575247.489279:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575247.489288:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575247.489293:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575247.489298:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575247.489307:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516798592, portal 25 00000100:00000001:0.0:1788575247.489311:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575247.489313:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788575247.489321:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516798592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.489335:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575247.489339:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575247.489343:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575247.489346:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516798592, offset 0 00000400:00000200:0.0:1788575247.489365:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575247.489376:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575247.489384:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575247.489401:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575247.489407:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575247.489413:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575247.489419:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575247.489426:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547891:547891:256:4294967295] 192.168.202.128@tcp LPNI seq info [547814:547814:8:4294967295] 00000400:00000200:0.0:1788575247.489433:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575247.489446:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575247.489454:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575247.489462:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575247.489473:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575247.489478:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788575247.489498:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575247.489508:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575247.489515:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575247.489524:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575247.489530:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575247.489533:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575247.489545:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575247.489555:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516798592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.489576:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575247.489585:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516798592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.489601:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.489603:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575247.489608:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575247.489615:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575247.489619:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310611181 00000100:00000010:0.0:1788575247.489626:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575247.489634:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575247.489641:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310611181 00000100:00000010:0.0:1788575247.489648:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50a80. 00000100:00000001:0.0:1788575247.489652:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575247.489656:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575247.489659:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.489663:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575247.489667:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575247/real 1788575247] req@ffff8fc4ac822300 x1875440516798592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575247.489674:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575247.489677:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575247.489681:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575247.489683:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575247.489686:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575247.489689:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575247.489703:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516798592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575247.489726:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516798592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.489743:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575247.489749:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.489756:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575247.489759:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575247.489775:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516798592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.489789:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575247.489797:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516798592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.489850:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575247.489855:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.489858:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575247.489861:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.489866:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575247.489869:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575247.489872:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575247.489876:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575247.489880:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575247.489886:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516798592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575247.489899:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516798592:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575247.489904:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.489910:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575247.489915:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516798592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.489938:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575247.489941:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575247.489945:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575247.489948:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575247.489952:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575247.489955:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575247.489959:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575247.489961:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788575247.489965:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575247.489968:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575247.489972:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788575247.489975:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575247.489977:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.489984:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.489990:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575247.489993:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.489998:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575247.490001:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575247.490004:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575247.490013:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516798720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490030:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575247.490034:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575247.490039:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575247.490043:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575247.490048:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575247.490053:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575247.490059:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575247.490061:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575247.490063:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575247.490067:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575247.490071:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575247.490074:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575247.490078:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575247.490088:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575247.490092:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575247.490097:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516798720, portal 10 00000100:00000001:0.0:1788575247.490100:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575247.490102:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788575247.490113:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516798720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490124:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575247.490130:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575247.490132:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575247.490137:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516798720, offset 0 00000400:00000200:0.0:1788575247.490145:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575247.490151:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575247.490158:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575247.490162:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575247.490165:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575247.490170:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575247.490174:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575247.490179:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547892:547892:256:4294967295] 192.168.202.128@tcp LPNI seq info [547815:547815:8:4294967295] 00000400:00000200:0.0:1788575247.490187:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575247.490200:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575247.490209:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575247.490214:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575247.490223:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575247.490229:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788575247.490253:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575247.490257:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575247.490263:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575247.490270:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575247.490274:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575247.490277:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575247.490289:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575247.490295:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516798720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490305:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575247.490310:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516798720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490318:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.490319:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575247.490323:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575247.490328:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575247.490335:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575247.490339:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310607379 00000100:00000001:0.0:1788575247.490342:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310611181 00000100:00000001:0.0:1788575247.490347:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310607379 00000100:00000001:0.0:1788575247.490352:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310611181 00000100:00000001:0.0:1788575247.490355:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575247.490358:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.490361:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575247.490366:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575247/real 1788575247] req@ffff8fc4ac820700 x1875440516798720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575247.490383:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575247.490401:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575247.490403:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575247.490406:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575247.490410:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575247.490413:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575247.490417:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516798720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575247.490424:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516798720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490436:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575247.490438:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.490440:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575247.490441:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575247.490445:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516798720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490450:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575247.490453:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516798720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490458:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575247.490459:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.490460:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575247.490461:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.490462:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575247.490463:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575247.490465:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575247.490467:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575247.490468:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575247.490471:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516798720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575247.490476:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575247.490478:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575247.490480:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516798848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490485:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575247.490486:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575247.490490:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575247.490492:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575247.490493:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575247.490495:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798848:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575247.490497:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575247.490499:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575247.490500:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575247.490501:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575247.490503:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575247.490505:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575247.490507:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575247.490509:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575247.490511:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575247.490513:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516798848, portal 10 00000100:00000001:0.0:1788575247.490515:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575247.490516:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788575247.490521:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516798848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490525:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575247.490526:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575247.490528:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575247.490529:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516798848, offset 0 00000400:00000200:0.0:1788575247.490532:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575247.490534:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575247.490537:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575247.490539:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575247.490540:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575247.490542:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575247.490543:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575247.490546:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547893:547893:256:4294967295] 192.168.202.128@tcp LPNI seq info [547816:547816:8:4294967295] 00000400:00000200:0.0:1788575247.490551:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575247.490557:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575247.490568:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575247.490572:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575247.490576:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575247.490579:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788575247.490590:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575247.490592:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575247.490595:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575247.490599:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575247.490602:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575247.490603:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575247.490606:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575247.490608:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516798848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490614:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575247.490616:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516798848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490620:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.490621:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575247.490624:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575247.490626:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575247.490628:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575247.490630:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310607386 00000100:00000001:0.0:1788575247.490631:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310611181 00000100:00000001:0.0:1788575247.490633:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310607386 00000100:00000001:0.0:1788575247.490634:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310611181 00000100:00000001:0.0:1788575247.490636:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575247.490637:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.490639:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575247.490641:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575247/real 1788575247] req@ffff8fc4ac823480 x1875440516798848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575247.490651:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575247.490652:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575247.490653:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575247.490655:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575247.490656:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575247.490657:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575247.490660:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516798848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575247.490665:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516798848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490670:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575247.490670:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.490672:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575247.490672:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575247.490675:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516798848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490679:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575247.490681:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516798848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490685:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575247.490687:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.490688:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575247.490689:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.490690:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575247.490692:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575247.490694:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575247.490696:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575247.490697:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575247.490705:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516798848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575247.490713:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798848:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575247.490717:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.490719:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575247.490723:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516798720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490730:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575247.490732:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575247.490734:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575247.490737:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575247.490739:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575247.490740:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575247.490742:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575247.490744:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788575247.490747:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575247.490748:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575247.490750:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788575247.490752:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575247.490753:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.490754:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575247.490757:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516798848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490761:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575247.490762:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575247.490763:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575247.490765:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575247.490765:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575247.490766:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575247.490767:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575247.490768:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788575247.490769:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575247.490774:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575247.490775:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788575247.490777:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575247.490778:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.490779:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.490782:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575247.490783:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.490784:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575247.490786:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575247.490787:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575247.490789:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516798976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490794:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575247.490795:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575247.490797:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575247.490798:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575247.490799:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575247.490801:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575247.490803:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575247.490804:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575247.490805:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575247.490807:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575247.490828:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575247.490830:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575247.490832:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575247.490834:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575247.490837:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575247.490840:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516798976, portal 4 00000100:00000001:0.0:1788575247.490842:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575247.490843:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788575247.490848:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516798976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490852:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575247.490854:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575247.490859:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575247.490860:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516798976, offset 0 00000400:00000200:0.0:1788575247.490863:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575247.490866:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575247.490868:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575247.490871:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575247.490872:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575247.490874:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575247.490876:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575247.490878:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547894:547894:256:4294967295] 192.168.202.128@tcp LPNI seq info [547817:547817:8:4294967295] 00000400:00000200:0.0:1788575247.490880:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575247.490885:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575247.490888:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575247.490889:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575247.490892:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575247.490894:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788575247.490901:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575247.490902:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575247.490904:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575247.490907:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575247.490909:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575247.490910:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575247.490912:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575247.490914:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516798976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490919:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575247.490921:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516798976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490926:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.490927:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575247.490934:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575247.490936:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575247.490939:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575247.490940:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310607527 00000100:00000001:0.0:1788575247.490942:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310611181 00000100:00000001:0.0:1788575247.490943:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310607527 00000100:00000001:0.0:1788575247.490945:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310611181 00000100:00000001:0.0:1788575247.490946:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575247.490947:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.490948:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575247.490951:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575247/real 1788575247] req@ffff8fc4ac821f80 x1875440516798976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575247.490956:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575247.490956:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575247.490958:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575247.490959:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575247.490960:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575247.490962:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575247.490963:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516798976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575247.490969:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516798976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490972:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575247.490973:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.490975:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575247.490976:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575247.490978:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516798976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490982:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575247.490985:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516798976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.490993:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575247.490995:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.490997:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575247.490998:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.491000:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575247.491002:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575247.491004:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575247.491008:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575247.491010:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575247.491014:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516798976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575247.491023:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516798976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575247.491028:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575247.491031:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516799104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.491038:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575247.491040:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575247.491042:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575247.491044:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575247.491046:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575247.491048:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516799104:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575247.491050:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575247.491051:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575247.491053:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575247.491054:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575247.491056:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575247.491058:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575247.491060:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575247.491066:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575247.491068:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575247.491071:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516799104, portal 4 00000100:00000001:0.0:1788575247.491072:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575247.491073:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788575247.491077:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516799104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.491082:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575247.491083:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575247.491085:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575247.491087:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516799104, offset 0 00000400:00000200:0.0:1788575247.491089:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575247.491092:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575247.491094:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575247.491096:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575247.491098:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575247.491099:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575247.491101:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575247.491103:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547895:547895:256:4294967295] 192.168.202.128@tcp LPNI seq info [547818:547818:8:4294967295] 00000400:00000200:0.0:1788575247.491105:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575247.491109:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575247.491112:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575247.491114:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575247.491117:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575247.491119:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788575247.491126:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575247.491127:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575247.491129:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575247.491132:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575247.491134:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575247.491135:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575247.491140:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575247.491142:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516799104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.491148:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575247.491150:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516799104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.491154:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.491155:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575247.491157:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575247.491159:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575247.491162:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575247.491163:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310607529 00000100:00000001:0.0:1788575247.491164:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310611181 00000100:00000001:0.0:1788575247.491166:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310607529 00000100:00000001:0.0:1788575247.491167:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310611181 00000100:00000001:0.0:1788575247.491168:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575247.491170:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.491171:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575247.491173:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575247/real 1788575247] req@ffff8fc4ac822d80 x1875440516799104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575247.491178:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575247.491179:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575247.491180:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575247.491182:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575247.491183:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575247.491184:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575247.491186:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516799104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575247.491191:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516799104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.491202:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575247.491203:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.491205:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575247.491206:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575247.491208:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516799104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.491213:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575247.491215:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516799104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.491220:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575247.491222:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.491224:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575247.491225:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.491226:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575247.491228:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575247.491231:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575247.491234:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575247.491236:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575247.491241:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516799104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575247.491249:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516799104:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575247.491253:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.491256:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575247.491259:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516798976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.491267:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575247.491268:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575247.491275:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575247.491277:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575247.491278:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575247.491279:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575247.491281:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575247.491282:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788575247.491283:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575247.491284:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575247.491285:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788575247.491287:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575247.491288:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.491289:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575247.491292:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516799104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575247.491296:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575247.491297:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575247.491298:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575247.491300:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575247.491301:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575247.491301:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575247.491302:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575247.491303:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788575247.491304:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575247.491305:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575247.491306:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788575247.491307:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575247.491308:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.491309:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575247.491311:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575247.491312:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575247.491313:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575247.491314:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575248.000890:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575248.000905:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575248.000911:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575248.000919:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575248.000926:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575248.000954:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575248.000961:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575248.000965:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575248.000968:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575248.001004:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575248.001008:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575248.001010:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575248.001012:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575248.001015:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575248.001017:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575248.001020:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575248.001021:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575248.001024:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575248.705004:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575248.705009:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575248.705012:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575249.024907:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575249.024914:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575249.024920:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575249.024928:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575249.024937:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575249.024942:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575249.024951:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575249.024954:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575249.024957:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575249.024972:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575249.024975:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575249.024976:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575249.024979:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575249.024982:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575249.024985:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575249.024987:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575249.024989:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575249.024991:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575250.048914:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575250.048921:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575250.048924:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575250.048931:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575250.048956:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575250.048962:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575250.048976:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575250.048980:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575250.048984:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575250.753006:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575250.753010:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575250.753013:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575251.072920:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575251.072929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575251.072934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575251.072942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575251.072948:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575251.072952:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575251.072974:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575251.072979:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575251.072986:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575251.328907:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310611181, at 4310611200 00000100:00000010:0.0:1788575251.328924:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575251.776996:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575251.776999:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575251.777001:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575252.096900:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575252.096910:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575252.096914:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575252.096921:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575252.096928:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575252.096934:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.096941:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575252.096946:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.096951:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.096982:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575252.096985:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575252.096986:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575252.096988:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575252.096991:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575252.096994:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.097011:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575252.097012:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.097014:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.609130:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575252.609136:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575252.609139:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575252.609147:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6680 x1875440516799232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.609161:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575252.609164:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575252.609168:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575252.609171:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575252.609185:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575252.609192:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516799232:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575252.609199:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575252.609205:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575252.609207:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575252.609212:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575252.609221:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575252.609225:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575252.609236:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575252.609240:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575252.609248:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575252.609255:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516799232, portal 25 00000100:00000001:0.0:1788575252.609260:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575252.609263:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605330560 : -123400104221056 : ffff8fc4ac9f6680) 00000100:00000040:0.0:1788575252.609272:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6680 x1875440516799232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.609284:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575252.609288:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575252.609293:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575252.609296:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516799232, offset 0 00000400:00000200:0.0:1788575252.609308:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575252.609321:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575252.609339:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575252.609348:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575252.609357:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575252.609365:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575252.609372:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575252.609379:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547896:547896:256:4294967295] 192.168.202.128@tcp LPNI seq info [547819:547819:8:4294967295] 00000400:00000200:0.0:1788575252.609384:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575252.609395:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575252.609403:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575252.609409:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575252.609419:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575252.609425:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788575252.609447:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575252.609449:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575252.609455:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575252.609464:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575252.609471:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575252.609476:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575252.609483:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575252.609490:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6680 x1875440516799232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.609507:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575252.609512:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6680 x1875440516799232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.609523:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.609527:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575252.609533:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575252.609539:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575252.609544:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310616301 00000100:00000010:0.0:1788575252.609550:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575252.609565:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575252.609571:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310616301 00000100:00000010:0.0:1788575252.609576:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50e40. 00000100:00000001:0.0:1788575252.609582:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575252.609591:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575252.609598:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.609604:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575252.609616:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575252/real 1788575252] req@ffff8fc4ac9f6680 x1875440516799232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575252.609640:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575252.609643:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575252.609646:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575252.609650:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575252.609653:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575252.609659:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575252.609667:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6680 x1875440516799232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575252.609688:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6680 x1875440516799232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.609701:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575252.609705:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.609710:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575252.609713:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575252.609720:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6680 x1875440516799232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.609737:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575252.609746:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6680 x1875440516799232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.609784:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575252.609791:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.609803:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575252.609806:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.609829:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575252.609831:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575252.609836:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575252.609845:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575252.609848:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575252.609857:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6680 x1875440516799232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575252.609880:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516799232:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575252.609902:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.609908:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575252.609916:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6680 x1875440516799232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.609931:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575252.609935:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575252.609939:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575252.609945:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575252.609949:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575252.609953:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575252.609958:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575252.609962:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788575252.609967:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575252.609970:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575252.609973:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6680. 00000100:00000001:0.0:1788575252.609978:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575252.609980:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.609982:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.609990:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575252.609993:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.609996:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575252.610001:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575252.610004:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575252.610017:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4000 x1875440516799360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610023:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575252.610025:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575252.610029:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575252.610031:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575252.610034:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575252.610039:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516799360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575252.610048:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575252.610051:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575252.610054:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575252.610057:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575252.610062:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575252.610065:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575252.610073:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575252.610077:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575252.610082:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575252.610086:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516799360, portal 10 00000100:00000001:0.0:1788575252.610089:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575252.610090:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605320704 : -123400104230912 : ffff8fc4ac9f4000) 00000100:00000040:0.0:1788575252.610097:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4000 x1875440516799360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610102:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575252.610104:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575252.610106:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575252.610108:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516799360, offset 0 00000400:00000200:0.0:1788575252.610110:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575252.610113:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575252.610115:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575252.610118:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575252.610120:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575252.610121:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575252.610127:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575252.610129:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547897:547897:256:4294967295] 192.168.202.128@tcp LPNI seq info [547820:547820:8:4294967295] 00000400:00000200:0.0:1788575252.610132:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575252.610137:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575252.610141:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575252.610143:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575252.610146:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575252.610148:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788575252.610155:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575252.610157:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575252.610159:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575252.610161:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575252.610164:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575252.610165:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575252.610167:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575252.610169:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4000 x1875440516799360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610191:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575252.610194:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4000 x1875440516799360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610199:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.610200:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575252.610203:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575252.610206:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575252.610208:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575252.610210:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310612499 00000100:00000001:0.0:1788575252.610212:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310616301 00000100:00000001:0.0:1788575252.610213:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310612499 00000100:00000001:0.0:1788575252.610215:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310616301 00000100:00000001:0.0:1788575252.610222:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575252.610223:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.610225:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575252.610228:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575252/real 1788575252] req@ffff8fc4ac9f4000 x1875440516799360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575252.610234:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575252.610234:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575252.610236:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575252.610237:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575252.610239:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575252.610240:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575252.610242:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4000 x1875440516799360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575252.610248:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4000 x1875440516799360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610252:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575252.610253:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610255:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575252.610255:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575252.610258:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4000 x1875440516799360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610262:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575252.610264:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4000 x1875440516799360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610268:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575252.610270:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610271:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575252.610271:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610273:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575252.610274:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575252.610289:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575252.610291:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575252.610293:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575252.610296:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4000 x1875440516799360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575252.610301:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516799360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575252.610303:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575252.610305:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7100 x1875440516799488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610309:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575252.610310:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575252.610312:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575252.610313:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575252.610314:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575252.610316:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516799488:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575252.610318:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575252.610320:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575252.610321:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575252.610322:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575252.610324:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575252.610325:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575252.610328:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575252.610329:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575252.610331:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575252.610333:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516799488, portal 10 00000100:00000001:0.0:1788575252.610335:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575252.610336:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605333248 : -123400104218368 : ffff8fc4ac9f7100) 00000100:00000040:0.0:1788575252.610339:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7100 x1875440516799488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610351:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575252.610352:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575252.610354:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575252.610356:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516799488, offset 0 00000400:00000200:0.0:1788575252.610358:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575252.610361:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575252.610363:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575252.610365:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575252.610367:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575252.610368:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575252.610370:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575252.610372:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547898:547898:256:4294967295] 192.168.202.128@tcp LPNI seq info [547821:547821:8:4294967295] 00000400:00000200:0.0:1788575252.610375:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575252.610379:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575252.610382:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575252.610384:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575252.610386:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575252.610388:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788575252.610396:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575252.610397:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575252.610399:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575252.610401:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575252.610403:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575252.610404:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575252.610406:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575252.610408:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7100 x1875440516799488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610414:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575252.610416:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7100 x1875440516799488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610427:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.610429:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575252.610431:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575252.610433:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575252.610436:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575252.610437:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310612506 00000100:00000001:0.0:1788575252.610439:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310616301 00000100:00000001:0.0:1788575252.610440:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310612506 00000100:00000001:0.0:1788575252.610442:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310616301 00000100:00000001:0.0:1788575252.610443:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575252.610444:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.610445:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575252.610448:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575252/real 1788575252] req@ffff8fc4ac9f7100 x1875440516799488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575252.610452:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575252.610453:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575252.610454:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575252.610456:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575252.610457:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575252.610459:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575252.610461:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7100 x1875440516799488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575252.610466:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7100 x1875440516799488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610469:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575252.610470:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610471:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575252.610472:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575252.610474:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7100 x1875440516799488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610482:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575252.610485:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7100 x1875440516799488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610489:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575252.610490:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610491:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575252.610492:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610493:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575252.610495:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575252.610497:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575252.610498:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575252.610500:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575252.610502:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7100 x1875440516799488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575252.610507:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516799488:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575252.610509:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575252.610511:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7800 x1875440516799616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610516:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575252.610517:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575252.610518:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575252.610519:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575252.610520:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575252.610522:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516799616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575252.610524:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575252.610525:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575252.610529:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575252.610530:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575252.610532:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575252.610533:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575252.610536:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575252.610537:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575252.610540:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575252.610542:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516799616, portal 4 00000100:00000001:0.0:1788575252.610543:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575252.610544:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335040 : -123400104216576 : ffff8fc4ac9f7800) 00000100:00000040:0.0:1788575252.610548:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7800 x1875440516799616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610552:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575252.610554:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575252.610555:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575252.610556:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516799616, offset 0 00000400:00000200:0.0:1788575252.610559:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575252.610561:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575252.610563:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575252.610565:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575252.610567:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575252.610568:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575252.610570:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575252.610572:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547899:547899:256:4294967295] 192.168.202.128@tcp LPNI seq info [547822:547822:8:4294967295] 00000400:00000200:0.0:1788575252.610575:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575252.610580:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575252.610585:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575252.610588:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575252.610592:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575252.610594:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788575252.610605:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575252.610614:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575252.610617:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575252.610621:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575252.610625:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575252.610627:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575252.610630:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575252.610634:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7800 x1875440516799616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610640:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575252.610642:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7800 x1875440516799616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610647:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.610648:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575252.610650:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575252.610652:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575252.610655:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575252.610656:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310612646 00000100:00000001:0.0:1788575252.610658:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310616301 00000100:00000001:0.0:1788575252.610659:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310612646 00000100:00000001:0.0:1788575252.610661:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310616301 00000100:00000001:0.0:1788575252.610662:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575252.610663:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.610664:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575252.610667:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575252/real 1788575252] req@ffff8fc4ac9f7800 x1875440516799616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575252.610672:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575252.610673:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575252.610674:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575252.610676:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575252.610677:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575252.610683:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575252.610685:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7800 x1875440516799616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575252.610690:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7800 x1875440516799616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610694:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575252.610695:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610696:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575252.610697:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575252.610699:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7800 x1875440516799616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610704:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575252.610707:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7800 x1875440516799616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610711:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575252.610712:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610713:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575252.610714:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610715:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575252.610717:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575252.610718:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575252.610720:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575252.610721:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575252.610723:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7800 x1875440516799616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575252.610728:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516799616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575252.610730:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610731:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575252.610736:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4000 x1875440516799360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610741:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575252.610742:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575252.610744:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575252.610745:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575252.610746:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575252.610747:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575252.610749:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575252.610750:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35800. 02000000:00000001:0.0:1788575252.610752:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575252.610753:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575252.610754:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4000. 00000100:00000001:0.0:1788575252.610755:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575252.610756:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610758:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575252.610760:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7100 x1875440516799488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610764:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575252.610765:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575252.610766:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575252.610767:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575252.610768:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575252.610769:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575252.610770:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575252.610771:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36400. 02000000:00000001:0.0:1788575252.610772:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575252.610773:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575252.610774:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7100. 00000100:00000001:0.0:1788575252.610775:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575252.610776:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610777:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575252.610779:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7800 x1875440516799616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610782:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575252.610786:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575252.610787:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575252.610788:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575252.610790:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575252.610790:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575252.610792:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575252.610793:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34400. 02000000:00000001:0.0:1788575252.610794:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575252.610794:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575252.610795:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7800. 00000100:00000001:0.0:1788575252.610796:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575252.610797:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610798:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.610801:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575252.610802:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.610803:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575252.610804:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575252.610805:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575252.610822:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5180 x1875440516799744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610827:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575252.610828:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575252.610829:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575252.610831:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575252.610832:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575252.610834:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516799744:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575252.610836:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575252.610837:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575252.610838:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575252.610839:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575252.610841:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575252.610843:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575252.610845:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575252.610847:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575252.610849:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575252.610852:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516799744, portal 4 00000100:00000001:0.0:1788575252.610856:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575252.610857:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605325184 : -123400104226432 : ffff8fc4ac9f5180) 00000100:00000040:0.0:1788575252.610862:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5180 x1875440516799744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610869:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575252.610871:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575252.610874:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575252.610877:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516799744, offset 0 00000400:00000200:0.0:1788575252.610880:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575252.610884:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575252.610888:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575252.610892:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575252.610894:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575252.610896:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575252.610899:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575252.610902:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547900:547900:256:4294967295] 192.168.202.128@tcp LPNI seq info [547823:547823:8:4294967295] 00000400:00000200:0.0:1788575252.610907:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575252.610914:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575252.610918:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575252.610922:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575252.610924:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575252.610927:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788575252.610938:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575252.610939:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575252.610941:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575252.610944:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575252.610946:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575252.610947:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575252.610949:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575252.610952:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5180 x1875440516799744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610964:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575252.610966:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5180 x1875440516799744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.610971:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.610972:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575252.610974:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575252.610976:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575252.610979:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575252.610980:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310612649 00000100:00000001:0.0:1788575252.610982:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310616301 00000100:00000001:0.0:1788575252.610983:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310612649 00000100:00000001:0.0:1788575252.610985:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310616301 00000100:00000001:0.0:1788575252.610986:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575252.610987:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.610989:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575252.610991:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575252/real 1788575252] req@ffff8fc4ac9f5180 x1875440516799744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575252.610996:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575252.610996:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575252.610998:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575252.610999:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575252.611000:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575252.611002:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575252.611004:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516799744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575252.611009:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516799744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.611013:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575252.611018:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.611019:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575252.611020:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575252.611023:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516799744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.611027:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575252.611029:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516799744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.611034:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575252.611036:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.611037:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575252.611037:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.611039:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575252.611040:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575252.611041:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575252.611043:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575252.611045:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575252.611047:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516799744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575252.611051:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516799744:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575252.611054:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.611055:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575252.611057:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516799744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575252.611061:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575252.611062:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575252.611064:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575252.611065:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575252.611066:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575252.611069:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575252.611071:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575252.611072:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788575252.611074:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575252.611075:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575252.611076:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788575252.611077:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575252.611078:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.611079:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575252.611082:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575252.611083:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575252.611084:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575252.611085:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575253.120919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575253.120926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575253.120931:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575253.120939:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575253.120947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575253.120952:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575253.120962:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575253.120965:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575253.120970:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575253.824992:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575253.824995:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575253.824998:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575254.145208:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575254.145212:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575254.145213:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575254.145216:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575254.145218:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575254.145220:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575254.145222:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575254.145224:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575254.145226:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575255.168947:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575255.168952:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575255.168953:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575255.168957:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575255.168974:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575255.168978:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575255.168984:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575255.168989:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575255.168996:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575255.873008:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575255.873013:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575255.873016:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575256.192913:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575256.192920:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575256.192925:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575256.192935:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575256.192942:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575256.192948:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575256.192971:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575256.192975:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575256.192979:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575256.448898:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310616301, at 4310616320 00000100:00000010:0.0:1788575256.448911:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575256.897019:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575256.897022:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575256.897024:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575257.216931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575257.216936:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575257.216939:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575257.216943:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575257.216949:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575257.216953:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.216962:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575257.216965:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.216969:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.729170:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575257.729175:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575257.729180:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575257.729188:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516799872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.729205:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575257.729216:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575257.729223:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575257.729227:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575257.729232:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575257.729239:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516799872:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575257.729247:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575257.729251:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575257.729255:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575257.729258:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575257.729262:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575257.729266:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575257.729273:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575257.729279:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575257.729289:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575257.729299:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516799872, portal 25 00000100:00000001:0.0:1788575257.729302:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575257.729303:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788575257.729308:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516799872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.729313:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575257.729316:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575257.729320:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575257.729324:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516799872, offset 0 00000400:00000200:0.0:1788575257.729333:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575257.729342:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575257.729350:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575257.729359:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575257.729364:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575257.729371:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575257.729376:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575257.729383:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547901:547901:256:4294967295] 192.168.202.128@tcp LPNI seq info [547824:547824:8:4294967295] 00000400:00000200:0.0:1788575257.729390:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575257.729404:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575257.729417:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575257.729423:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575257.729434:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575257.729441:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575257.729465:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575257.729469:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575257.729474:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575257.729482:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575257.729490:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575257.729495:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575257.729503:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575257.729511:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516799872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.729528:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575257.729533:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516799872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.729546:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.729551:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575257.729557:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575257.729566:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575257.729572:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310621421 00000100:00000010:0.0:1788575257.729577:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575257.729584:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575257.729592:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310621421 00000100:00000010:0.0:1788575257.729597:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50300. 00000100:00000001:0.0:1788575257.729602:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575257.729607:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575257.729611:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.729614:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575257.729619:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575257/real 1788575257] req@ffff8fc4ac9f6a00 x1875440516799872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575257.729647:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575257.729650:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575257.729653:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575257.729659:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575257.729663:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575257.729668:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575257.729677:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516799872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575257.729695:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516799872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.729711:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575257.729723:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.729730:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575257.729734:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575257.729741:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516799872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.729749:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575257.729752:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516799872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.729764:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575257.729767:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.729770:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575257.729772:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.729775:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575257.729777:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575257.729779:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575257.729787:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575257.729790:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575257.729795:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516799872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575257.729837:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516799872:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575257.729843:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.729846:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575257.729852:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516799872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.729869:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575257.729872:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575257.729876:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575257.729882:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575257.729885:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575257.729887:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575257.729890:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575257.729893:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36c00. 02000000:00000001:0.0:1788575257.729897:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575257.729900:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575257.729902:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788575257.729905:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575257.729906:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.729907:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.729914:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575257.729917:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.729921:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575257.729924:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575257.729926:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575257.729932:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4700 x1875440516800000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.729946:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575257.729949:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575257.729954:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575257.729957:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575257.729974:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575257.729984:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516800000:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575257.729998:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575257.730002:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575257.730005:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575257.730011:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575257.730015:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575257.730020:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575257.730028:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575257.730033:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575257.730037:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575257.730042:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516800000, portal 10 00000100:00000001:0.0:1788575257.730047:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575257.730050:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605322496 : -123400104229120 : ffff8fc4ac9f4700) 00000100:00000040:0.0:1788575257.730059:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4700 x1875440516800000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730068:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575257.730074:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575257.730077:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575257.730081:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516800000, offset 0 00000400:00000200:0.0:1788575257.730085:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575257.730094:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575257.730100:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575257.730105:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575257.730109:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575257.730112:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575257.730117:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575257.730124:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547902:547902:256:4294967295] 192.168.202.128@tcp LPNI seq info [547825:547825:8:4294967295] 00000400:00000200:0.0:1788575257.730131:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575257.730141:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575257.730147:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575257.730152:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575257.730159:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575257.730166:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575257.730184:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575257.730186:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575257.730188:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575257.730195:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575257.730202:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575257.730205:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575257.730208:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575257.730214:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4700 x1875440516800000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730234:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575257.730240:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4700 x1875440516800000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730250:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.730252:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575257.730255:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575257.730259:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575257.730262:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575257.730263:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310617619 00000100:00000001:0.0:1788575257.730266:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310621421 00000100:00000001:0.0:1788575257.730268:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310617619 00000100:00000001:0.0:1788575257.730269:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310621421 00000100:00000001:0.0:1788575257.730271:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575257.730272:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.730273:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575257.730276:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575257/real 1788575257] req@ffff8fc4ac9f4700 x1875440516800000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575257.730281:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575257.730282:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575257.730287:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575257.730289:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575257.730290:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575257.730292:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575257.730295:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4700 x1875440516800000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575257.730300:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4700 x1875440516800000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730304:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575257.730305:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730306:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575257.730307:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575257.730310:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4700 x1875440516800000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730314:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575257.730317:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4700 x1875440516800000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730321:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575257.730323:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730324:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575257.730325:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730326:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575257.730327:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575257.730329:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575257.730331:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575257.730332:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575257.730334:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4700 x1875440516800000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575257.730339:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516800000:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575257.730345:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575257.730347:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5f80 x1875440516800128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730352:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575257.730353:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575257.730354:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575257.730355:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575257.730357:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575257.730358:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516800128:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575257.730360:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575257.730362:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575257.730363:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575257.730364:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575257.730366:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575257.730368:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575257.730370:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575257.730371:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575257.730374:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575257.730376:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516800128, portal 10 00000100:00000001:0.0:1788575257.730377:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575257.730378:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605328768 : -123400104222848 : ffff8fc4ac9f5f80) 00000100:00000040:0.0:1788575257.730382:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5f80 x1875440516800128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730387:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575257.730388:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575257.730390:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575257.730391:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516800128, offset 0 00000400:00000200:0.0:1788575257.730394:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575257.730396:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575257.730398:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575257.730400:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575257.730405:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575257.730407:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575257.730409:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575257.730411:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547903:547903:256:4294967295] 192.168.202.128@tcp LPNI seq info [547826:547826:8:4294967295] 00000400:00000200:0.0:1788575257.730414:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575257.730418:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575257.730422:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575257.730424:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575257.730427:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575257.730429:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575257.730436:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575257.730437:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575257.730439:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575257.730441:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575257.730443:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575257.730445:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575257.730446:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575257.730449:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5f80 x1875440516800128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730454:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575257.730456:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5f80 x1875440516800128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730460:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.730461:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575257.730463:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575257.730465:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575257.730467:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575257.730468:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310617626 00000100:00000001:0.0:1788575257.730470:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310621421 00000100:00000001:0.0:1788575257.730471:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310617626 00000100:00000001:0.0:1788575257.730476:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310621421 00000100:00000001:0.0:1788575257.730477:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575257.730479:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.730480:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575257.730483:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575257/real 1788575257] req@ffff8fc4ac9f5f80 x1875440516800128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575257.730488:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575257.730489:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575257.730490:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575257.730492:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575257.730493:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575257.730494:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575257.730496:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5f80 x1875440516800128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575257.730502:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5f80 x1875440516800128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730506:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575257.730507:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730508:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575257.730509:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575257.730511:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5f80 x1875440516800128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730515:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575257.730517:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5f80 x1875440516800128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730521:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575257.730522:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730523:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575257.730524:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730528:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575257.730530:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575257.730532:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575257.730534:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575257.730536:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575257.730539:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5f80 x1875440516800128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575257.730544:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516800128:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575257.730546:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575257.730548:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5180 x1875440516800256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730552:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575257.730553:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575257.730554:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575257.730555:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575257.730556:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575257.730558:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516800256:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575257.730560:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575257.730561:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575257.730562:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575257.730563:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575257.730565:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575257.730566:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575257.730569:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575257.730570:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575257.730572:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575257.730575:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516800256, portal 4 00000100:00000001:0.0:1788575257.730576:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575257.730577:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605325184 : -123400104226432 : ffff8fc4ac9f5180) 00000100:00000040:0.0:1788575257.730581:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5180 x1875440516800256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730589:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575257.730591:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575257.730592:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575257.730594:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516800256, offset 0 00000400:00000200:0.0:1788575257.730596:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575257.730599:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575257.730601:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575257.730604:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575257.730605:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575257.730607:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575257.730608:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575257.730610:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547904:547904:256:4294967295] 192.168.202.128@tcp LPNI seq info [547827:547827:8:4294967295] 00000400:00000200:0.0:1788575257.730613:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575257.730617:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575257.730620:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575257.730622:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575257.730624:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575257.730626:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575257.730633:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575257.730634:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575257.730636:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575257.730638:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575257.730640:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575257.730641:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575257.730643:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575257.730645:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5180 x1875440516800256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730650:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575257.730652:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5180 x1875440516800256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730664:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.730665:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575257.730668:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575257.730670:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575257.730672:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575257.730674:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310617766 00000100:00000001:0.0:1788575257.730675:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310621421 00000100:00000001:0.0:1788575257.730677:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310617766 00000100:00000001:0.0:1788575257.730679:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310621421 00000100:00000001:0.0:1788575257.730680:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575257.730681:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.730683:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575257.730685:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575257/real 1788575257] req@ffff8fc4ac9f5180 x1875440516800256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575257.730690:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575257.730690:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575257.730691:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575257.730693:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575257.730694:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575257.730695:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575257.730697:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516800256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575257.730703:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516800256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730707:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575257.730707:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730709:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575257.730709:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575257.730711:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516800256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730719:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575257.730722:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516800256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730726:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575257.730728:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730729:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575257.730729:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730731:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575257.730732:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575257.730733:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575257.730735:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575257.730737:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575257.730739:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516800256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575257.730744:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516800256:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575257.730746:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730748:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575257.730750:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4700 x1875440516800000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730754:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575257.730755:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575257.730756:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575257.730757:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575257.730758:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575257.730759:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575257.730760:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575257.730761:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34c00. 02000000:00000001:0.0:1788575257.730766:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575257.730767:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575257.730768:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4700. 00000100:00000001:0.0:1788575257.730770:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575257.730771:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730772:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575257.730775:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5f80 x1875440516800128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730779:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575257.730780:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575257.730781:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575257.730782:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575257.730783:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575257.730784:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575257.730785:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575257.730785:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35000. 02000000:00000001:0.0:1788575257.730787:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575257.730787:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575257.730788:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5f80. 00000100:00000001:0.0:1788575257.730790:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575257.730790:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730791:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575257.730793:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516800256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730797:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575257.730797:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575257.730798:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575257.730800:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575257.730802:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575257.730803:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575257.730804:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575257.730805:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35c00. 02000000:00000001:0.0:1788575257.730827:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575257.730829:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575257.730830:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788575257.730833:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575257.730840:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730843:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.730847:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575257.730848:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.730850:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575257.730852:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575257.730853:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575257.730858:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7800 x1875440516800384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730867:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575257.730868:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575257.730870:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575257.730871:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575257.730872:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575257.730874:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516800384:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575257.730877:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575257.730878:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575257.730879:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575257.730881:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575257.730883:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575257.730884:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575257.730887:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575257.730889:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575257.730891:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575257.730893:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516800384, portal 4 00000100:00000001:0.0:1788575257.730895:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575257.730896:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335040 : -123400104216576 : ffff8fc4ac9f7800) 00000100:00000040:0.0:1788575257.730899:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7800 x1875440516800384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730904:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575257.730905:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575257.730906:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575257.730908:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516800384, offset 0 00000400:00000200:0.0:1788575257.730911:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575257.730918:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575257.730921:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575257.730923:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575257.730924:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575257.730926:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575257.730928:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575257.730930:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547905:547905:256:4294967295] 192.168.202.128@tcp LPNI seq info [547828:547828:8:4294967295] 00000400:00000200:0.0:1788575257.730933:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575257.730937:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575257.730940:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575257.730942:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575257.730945:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575257.730947:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575257.730954:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575257.730955:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575257.730957:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575257.730960:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575257.730978:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575257.730981:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575257.730985:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575257.730988:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7800 x1875440516800384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.730996:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575257.731000:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7800 x1875440516800384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.731006:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.731008:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575257.731011:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575257.731014:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575257.731016:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575257.731023:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310617769 00000100:00000001:0.0:1788575257.731024:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310621421 00000100:00000001:0.0:1788575257.731026:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310617769 00000100:00000001:0.0:1788575257.731027:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310621421 00000100:00000001:0.0:1788575257.731029:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575257.731031:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.731032:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575257.731035:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575257/real 1788575257] req@ffff8fc4ac9f7800 x1875440516800384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575257.731040:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575257.731041:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575257.731042:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575257.731044:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575257.731045:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575257.731047:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575257.731049:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7800 x1875440516800384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575257.731054:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7800 x1875440516800384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.731058:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575257.731059:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.731060:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575257.731061:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575257.731063:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7800 x1875440516800384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.731067:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575257.731070:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7800 x1875440516800384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.731074:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575257.731079:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.731080:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575257.731081:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.731083:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575257.731084:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575257.731086:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575257.731088:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575257.731090:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575257.731092:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7800 x1875440516800384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575257.731098:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516800384:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575257.731100:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.731101:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575257.731103:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7800 x1875440516800384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575257.731107:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575257.731108:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575257.731109:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575257.731111:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575257.731111:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575257.731112:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575257.731113:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575257.731114:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36c00. 02000000:00000001:0.0:1788575257.731116:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575257.731117:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575257.731117:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7800. 00000100:00000001:0.0:1788575257.731119:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575257.731120:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.731121:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575257.731124:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575257.731125:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575257.731126:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575257.731130:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575258.240970:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575258.240977:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575258.240980:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575258.240987:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575258.240994:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575258.240998:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575258.241004:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575258.241007:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575258.241011:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575258.945010:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575258.945013:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575258.945015:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575259.264958:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575259.264963:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575259.264966:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575259.264973:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575259.264977:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575259.264980:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575259.264983:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575259.264987:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575259.264989:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575260.289077:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575260.289080:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575260.289082:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575260.289088:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575260.289092:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575260.289096:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575260.289099:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575260.289102:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575260.289107:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575260.993000:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575260.993004:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575260.993007:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575261.312932:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575261.312942:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575261.312947:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575261.312956:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575261.312977:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575261.312980:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575261.313008:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575261.313012:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575261.313015:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575261.568922:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310621421, at 4310621440 00000100:00000010:0.0:1788575261.568938:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575262.016966:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575262.016969:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575262.016972:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575262.336917:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575262.336929:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575262.336932:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575262.336938:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575262.336944:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575262.336949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575262.336954:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575262.336959:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575262.336963:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.336996:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575262.336999:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575262.337004:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575262.337010:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575262.337015:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575262.337018:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575262.337020:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575262.337021:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575262.337022:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575262.848925:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575262.848934:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575262.848938:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575262.848948:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575262.848951:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575262.848960:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15950 00000400:00000200:0.0:1788575262.848973:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575262.848996:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575262.848998:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575262.849001:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849004:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575262.849006:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849008:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575262.849011:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575262.849013:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575262.849015:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575262.849017:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575262.849019:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575262.849021:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575262.849023:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849024:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575262.849025:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849027:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575262.849029:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575262.849031:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849033:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575262.849035:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575262.849038:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575262.849040:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.849042:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575262.849043:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575262.849049:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575262.849052:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575262.849055:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575262.849056:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.849059:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575262.849061:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575262.849064:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575262.849070:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575262.849079:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575262.849087:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516800512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575262.849095:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575262.849099:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575262.849103:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516800512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575262.849125:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575262.849129:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575262.849133:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575262.849134:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575262.849136:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575262.849137:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575262.849141:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15950 00000400:00000200:0.0:1788575262.849147:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575262.849153:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575262.849159:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575262.849165:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849169:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575262.849175:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849177:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575262.849183:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575262.849185:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575262.849189:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575262.849193:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575262.849196:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575262.849200:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575262.849206:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849210:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575262.849221:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849224:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575262.849229:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575262.849234:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849236:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575262.849242:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575262.849249:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575262.849255:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.849257:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575262.849262:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575262.849265:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.849268:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575262.849272:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575262.849275:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575262.849277:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575262.849278:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.849279:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575262.849281:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575262.849285:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575262.849291:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575262.849297:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575262.849306:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516800640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575262.849321:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575262.849325:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575262.849332:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516800640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575262.849345:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575262.849351:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575262.849355:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575262.849358:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575262.849367:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575262.849373:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575262.849381:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15950 00000400:00000200:0.0:1788575262.849393:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575262.849400:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575262.849403:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575262.849411:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849418:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575262.849427:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849432:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575262.849439:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575262.849443:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575262.849448:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575262.849451:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575262.849457:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575262.849464:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575262.849473:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849477:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575262.849483:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849487:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575262.849493:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575262.849497:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849503:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575262.849507:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575262.849511:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575262.849516:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.849520:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575262.849523:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575262.849530:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.849534:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575262.849553:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575262.849557:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575262.849570:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575262.849575:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.849581:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575262.849584:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575262.849588:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575262.849598:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575262.849605:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575262.849615:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516800768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575262.849627:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575262.849632:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575262.849640:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516800768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575262.849658:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575262.849663:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575262.849668:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575262.849670:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575262.849672:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575262.849677:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575262.849684:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15950 00000400:00000200:0.0:1788575262.849712:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575262.849721:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575262.849726:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575262.849734:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849739:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575262.849745:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849769:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575262.849771:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575262.849779:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575262.849780:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575262.849783:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575262.849786:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575262.849791:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575262.849798:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849801:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575262.849804:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849837:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575262.849842:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575262.849846:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575262.849853:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575262.849857:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575262.849863:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575262.849866:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.849868:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575262.849871:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575262.849873:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.849877:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575262.849884:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575262.849889:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575262.849894:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575262.849897:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.849901:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575262.849904:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575262.849908:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575262.849915:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575262.849922:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575262.849932:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516800896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575262.849950:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575262.849954:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575262.849970:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516800896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575262.849989:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575262.849994:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575262.849999:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575262.850004:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575262.850006:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575262.850013:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575262.850019:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15950 00000400:00000200:0.0:1788575262.850029:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575262.850036:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575262.850042:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575262.850049:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.850055:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575262.850058:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.850064:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575262.850071:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575262.850075:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575262.850077:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575262.850082:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575262.850087:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575262.850091:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575262.850098:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.850103:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575262.850109:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575262.850115:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575262.850120:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575262.850125:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575262.850131:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575262.850142:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575262.850149:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575262.850155:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.850160:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575262.850165:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575262.850172:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.850178:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575262.850187:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575262.850193:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575262.850200:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575262.850205:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575262.850209:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575262.850214:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575262.850220:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575262.850226:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575262.850234:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575262.850244:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516801024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575262.850259:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575262.850263:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575262.850270:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516801024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575262.850305:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575262.850331:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15960) 00000020:00000001:0.0:1788575264.064987:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575264.064990:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575264.064992:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575264.384999:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575264.385005:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575264.385010:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575264.385016:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575264.385021:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575264.385025:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575264.385030:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575264.385047:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575264.385049:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575265.408913:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575265.408920:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575265.408923:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575265.408929:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575265.408934:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575265.408941:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575265.408962:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575265.408965:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575265.408967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575266.113003:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575266.113007:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575266.113011:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575266.433012:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575266.433024:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575266.433033:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575266.433047:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575266.433055:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575266.433062:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575266.433082:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575266.433086:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575266.433090:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575266.688892:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310626541, at 4310626560 00000100:00000010:0.0:1788575266.688912:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cff60. 00000020:00000001:0.0:1788575267.137093:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575267.137100:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575267.137106:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575267.456939:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575267.456948:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575267.456956:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575267.456974:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575267.456992:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575267.457010:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575267.457039:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575267.457042:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575267.457047:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575267.968937:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575267.968951:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575267.968954:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575267.968958:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575267.968963:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575267.968971:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15955 00000400:00000200:0.0:1788575267.968984:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575267.968993:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575267.968997:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575267.969001:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969004:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575267.969008:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969010:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575267.969014:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575267.969016:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575267.969019:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575267.969022:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575267.969024:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575267.969026:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575267.969029:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969031:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575267.969034:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969036:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575267.969039:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575267.969042:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969044:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575267.969046:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575267.969050:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575267.969052:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969054:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575267.969055:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575267.969084:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575267.969087:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575267.969091:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575267.969092:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969095:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575267.969098:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575267.969103:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575267.969113:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575267.969121:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575267.969136:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516801152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575267.969145:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575267.969149:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575267.969153:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516801152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575267.969178:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575267.969181:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575267.969184:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575267.969185:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575267.969186:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575267.969188:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575267.969190:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15955 00000400:00000200:0.0:1788575267.969192:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575267.969194:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575267.969196:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575267.969199:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969201:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575267.969203:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969205:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575267.969207:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575267.969208:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575267.969217:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575267.969221:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575267.969223:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575267.969227:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575267.969232:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969234:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575267.969239:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969243:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575267.969248:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575267.969252:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969257:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575267.969260:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575267.969268:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575267.969271:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969277:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575267.969282:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575267.969286:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969289:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575267.969295:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575267.969299:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575267.969303:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575267.969306:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969310:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575267.969312:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575267.969317:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575267.969320:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575267.969325:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575267.969332:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516801280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575267.969345:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575267.969347:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575267.969351:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516801280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575267.969366:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575267.969369:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575267.969373:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575267.969375:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575267.969377:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575267.969383:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575267.969388:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15955 00000400:00000200:0.0:1788575267.969394:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575267.969400:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575267.969404:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575267.969410:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969414:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575267.969418:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969421:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575267.969426:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575267.969429:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575267.969431:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575267.969435:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575267.969438:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575267.969441:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575267.969445:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969450:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575267.969455:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969457:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575267.969458:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575267.969460:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969461:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575267.969466:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575267.969468:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575267.969472:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969474:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575267.969476:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575267.969478:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969482:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575267.969488:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575267.969494:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575267.969499:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575267.969503:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969508:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575267.969512:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575267.969518:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575267.969527:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575267.969535:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575267.969547:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516801408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575267.969581:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575267.969585:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575267.969592:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516801408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575267.969609:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575267.969617:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575267.969624:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575267.969626:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575267.969628:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575267.969634:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575267.969641:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15955 00000400:00000200:0.0:1788575267.969651:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575267.969657:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575267.969661:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575267.969676:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969682:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575267.969688:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969691:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575267.969693:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575267.969696:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575267.969698:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575267.969701:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575267.969703:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575267.969707:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575267.969713:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969717:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575267.969721:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969726:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575267.969728:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575267.969731:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969736:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575267.969737:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575267.969742:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575267.969746:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969749:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575267.969752:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575267.969757:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969763:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575267.969766:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575267.969770:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575267.969775:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575267.969776:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969778:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575267.969779:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575267.969781:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575267.969785:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575267.969795:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575267.969802:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516801536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575267.969832:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575267.969834:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575267.969838:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516801536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575267.969844:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575267.969848:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575267.969852:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575267.969853:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575267.969855:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575267.969858:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575267.969863:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15955 00000400:00000200:0.0:1788575267.969868:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575267.969872:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575267.969874:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575267.969877:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969884:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575267.969889:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969896:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575267.969902:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575267.969907:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575267.969912:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575267.969916:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575267.969920:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575267.969924:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575267.969931:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969936:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575267.969943:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969946:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575267.969950:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575267.969957:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575267.969960:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575267.969963:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575267.969966:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575267.969969:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969975:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575267.969977:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575267.969981:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.969983:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575267.969987:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575267.969990:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575267.969994:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575267.969996:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575267.970000:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575267.970003:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575267.970005:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575267.970011:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575267.970016:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575267.970026:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516801664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575267.970035:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575267.970038:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575267.970045:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516801664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575267.970054:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575267.970059:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15965) 00000100:00000001:0.0:1788575268.480911:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575268.480919:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575268.480924:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575268.480948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575268.480954:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575268.480960:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575268.480978:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575268.480982:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575268.480985:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575269.185048:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575269.185051:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575269.185054:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575269.504931:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575269.504939:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575269.504942:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575269.504948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575269.504952:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575269.504956:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575269.504959:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575269.504962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575269.505168:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575270.528905:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575270.528912:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575270.528916:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575270.528924:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575270.528931:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575270.528936:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575270.528949:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575270.528953:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575270.528958:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575271.233061:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575271.233066:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575271.233070:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575271.552920:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575271.552929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575271.552934:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575271.552941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575271.552947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575271.552952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575271.552968:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575271.552973:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575271.552998:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575271.553035:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575271.553038:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575271.553040:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575271.553042:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575271.553044:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575271.553046:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575271.553048:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575271.553049:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575271.553050:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575271.808928:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310631661, at 4310631680 00000100:00000010:0.0:1788575271.808939:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf720. 00000020:00000001:0.0:1788575272.256954:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575272.256957:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575272.256958:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575272.576892:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575272.576896:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575272.576899:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575272.576906:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575272.576909:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575272.576913:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575272.576929:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575272.576933:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575272.576935:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575272.576952:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575272.576955:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575272.576956:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575272.576958:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575272.576960:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575272.576962:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575272.576964:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575272.576965:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575272.576967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.089112:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575273.089116:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575273.089119:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575273.089127:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516801792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.089147:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575273.089150:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575273.089157:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575273.089161:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575273.089167:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575273.089175:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516801792:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575273.089185:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575273.089192:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575273.089196:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575273.089203:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575273.089209:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575273.089213:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575273.089223:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575273.089229:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575273.089237:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575273.089246:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516801792, portal 25 00000100:00000001:0.0:1788575273.089251:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575273.089253:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575273.089262:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516801792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.089279:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575273.089284:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575273.089288:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575273.089292:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516801792, offset 0 00000400:00000200:0.0:1788575273.089301:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575273.089310:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575273.089319:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575273.089326:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575273.089331:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575273.089336:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575273.089340:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575273.089353:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547916:547916:256:4294967295] 192.168.202.128@tcp LPNI seq info [547839:547839:8:4294967295] 00000400:00000200:0.0:1788575273.089362:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575273.089376:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575273.089388:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575273.089395:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575273.089414:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575273.089420:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575273.089442:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575273.089445:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575273.089452:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575273.089460:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575273.089466:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575273.089472:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575273.089479:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575273.089484:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516801792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.089493:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575273.089499:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516801792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.089513:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.089519:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575273.089526:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575273.089535:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575273.089541:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310636781 00000100:00000010:0.0:1788575273.089549:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575273.089559:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575273.089567:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310636781 00000100:00000010:0.0:1788575273.089573:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50de0. 00000100:00000001:0.0:1788575273.089584:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575273.089589:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575273.089600:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.089605:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575273.089614:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575273/real 1788575273] req@ffff8fc4901a4e00 x1875440516801792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575273.089633:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575273.089636:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575273.089641:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575273.089647:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575273.089652:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575273.089656:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575273.089662:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516801792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575273.089682:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516801792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.089693:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575273.089696:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.089700:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575273.089703:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575273.089712:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516801792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.089747:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575273.089758:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516801792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.089772:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575273.089786:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.089790:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575273.089793:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.089798:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575273.089803:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575273.089827:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575273.089833:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575273.089846:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575273.089861:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516801792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575273.089881:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516801792:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575273.089891:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.089901:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575273.089907:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516801792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.089921:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575273.089925:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575273.089930:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575273.089936:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575273.089940:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575273.089943:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575273.089946:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575273.089951:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575273.089957:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575273.089960:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575273.089963:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575273.089971:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575273.089974:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.089979:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.089994:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575273.089997:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.090002:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575273.090004:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575273.090005:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575273.090011:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516801920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090019:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575273.090021:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575273.090023:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575273.090025:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575273.090034:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575273.090038:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516801920:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575273.090042:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575273.090045:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575273.090047:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575273.090049:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575273.090052:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575273.090055:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575273.090059:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575273.090062:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575273.090065:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575273.090068:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516801920, portal 10 00000100:00000001:0.0:1788575273.090070:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575273.090071:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575273.090077:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516801920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090083:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575273.090085:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575273.090087:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575273.090089:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516801920, offset 0 00000400:00000200:0.0:1788575273.090093:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575273.090097:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575273.090100:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575273.090103:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575273.090105:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575273.090108:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575273.090110:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575273.090113:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547917:547917:256:4294967295] 192.168.202.128@tcp LPNI seq info [547840:547840:8:4294967295] 00000400:00000200:0.0:1788575273.090118:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575273.090123:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575273.090128:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575273.090135:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575273.090139:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575273.090142:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575273.090153:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575273.090154:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575273.090157:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575273.090161:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575273.090164:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575273.090166:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575273.090169:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575273.090173:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516801920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090181:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575273.090184:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516801920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090190:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.090192:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575273.090196:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575273.090199:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575273.090203:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575273.090204:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310632979 00000100:00000001:0.0:1788575273.090207:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310636781 00000100:00000001:0.0:1788575273.090209:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310632979 00000100:00000001:0.0:1788575273.090211:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310636781 00000100:00000001:0.0:1788575273.090213:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575273.090214:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.090216:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575273.090219:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575273/real 1788575273] req@ffff8fc4901a6300 x1875440516801920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575273.090238:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575273.090239:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575273.090241:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575273.090243:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575273.090245:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575273.090247:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575273.090251:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516801920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575273.090259:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516801920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090265:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575273.090267:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.090269:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575273.090270:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575273.090274:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516801920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090280:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575273.090284:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516801920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090291:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575273.090293:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.090295:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575273.090296:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.090297:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575273.090299:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575273.090302:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575273.090305:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575273.090307:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575273.090316:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516801920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575273.090324:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516801920:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575273.090328:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575273.090331:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516802048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090337:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575273.090338:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575273.090340:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575273.090342:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575273.090344:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575273.090346:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802048:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575273.090350:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575273.090351:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575273.090353:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575273.090354:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575273.090358:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575273.090359:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575273.090363:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575273.090365:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575273.090368:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575273.090371:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516802048, portal 10 00000100:00000001:0.0:1788575273.090373:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575273.090375:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575273.090381:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516802048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090387:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575273.090390:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575273.090392:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575273.090394:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516802048, offset 0 00000400:00000200:0.0:1788575273.090413:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575273.090417:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575273.090421:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575273.090423:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575273.090426:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575273.090428:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575273.090430:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575273.090433:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547918:547918:256:4294967295] 192.168.202.128@tcp LPNI seq info [547841:547841:8:4294967295] 00000400:00000200:0.0:1788575273.090437:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575273.090443:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575273.090448:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575273.090451:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575273.090454:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575273.090457:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575273.090467:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575273.090469:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575273.090471:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575273.090475:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575273.090477:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575273.090479:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575273.090481:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575273.090485:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516802048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090492:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575273.090495:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516802048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090501:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.090503:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575273.090505:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575273.090508:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575273.090511:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575273.090516:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310632986 00000100:00000001:0.0:1788575273.090518:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310636781 00000100:00000001:0.0:1788575273.090521:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310632986 00000100:00000001:0.0:1788575273.090523:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310636781 00000100:00000001:0.0:1788575273.090525:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575273.090527:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.090529:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575273.090533:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575273/real 1788575273] req@ffff8fc4901a7480 x1875440516802048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575273.090540:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575273.090541:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575273.090543:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575273.090545:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575273.090547:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575273.090549:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575273.090552:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516802048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575273.090559:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516802048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090566:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575273.090567:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.090568:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575273.090570:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575273.090573:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516802048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090580:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575273.090583:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516802048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090589:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575273.090595:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.090597:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575273.090598:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.090599:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575273.090602:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575273.090604:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575273.090607:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575273.090610:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575273.090613:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516802048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575273.090621:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802048:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575273.090624:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575273.090627:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516802176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090634:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575273.090635:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575273.090637:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575273.090639:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575273.090641:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575273.090643:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802176:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575273.090646:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575273.090648:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575273.090649:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575273.090651:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575273.090654:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575273.090656:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575273.090659:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575273.090662:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575273.090665:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575273.090668:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516802176, portal 4 00000100:00000001:0.0:1788575273.090674:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575273.090675:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575273.090681:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516802176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090688:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575273.090690:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575273.090692:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575273.090694:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516802176, offset 0 00000400:00000200:0.0:1788575273.090697:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575273.090701:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575273.090704:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575273.090707:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575273.090709:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575273.090711:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575273.090714:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575273.090717:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547919:547919:256:4294967295] 192.168.202.128@tcp LPNI seq info [547842:547842:8:4294967295] 00000400:00000200:0.0:1788575273.090720:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575273.090726:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575273.090730:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575273.090733:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575273.090737:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575273.090749:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575273.090764:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575273.090766:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575273.090769:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575273.090772:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575273.090776:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575273.090777:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575273.090780:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575273.090784:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516802176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090801:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575273.090805:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516802176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090827:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.090829:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575273.090833:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575273.090836:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575273.090839:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575273.090841:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310633127 00000100:00000001:0.0:1788575273.090843:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310636781 00000100:00000001:0.0:1788575273.090845:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310633127 00000100:00000001:0.0:1788575273.090847:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310636781 00000100:00000001:0.0:1788575273.090849:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575273.090851:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.090853:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575273.090856:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575273/real 1788575273] req@ffff8fc4901a4700 x1875440516802176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575273.090863:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575273.090864:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575273.090866:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575273.090868:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575273.090870:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575273.090872:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575273.090875:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516802176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575273.090883:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516802176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090890:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575273.090896:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.090898:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575273.090899:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575273.090903:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516802176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090910:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575273.090913:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516802176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090920:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575273.090922:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.090923:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575273.090924:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.090926:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575273.090928:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575273.090930:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575273.090933:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575273.090935:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575273.090938:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516802176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575273.090945:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802176:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575273.090949:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575273.090952:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516802304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.090957:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575273.090959:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575273.090961:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575273.090963:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575273.090965:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575273.090972:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802304:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575273.090975:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575273.090977:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575273.090978:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575273.090980:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575273.090983:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575273.090984:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575273.090988:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575273.090990:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575273.090993:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575273.090996:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516802304, portal 4 00000100:00000001:0.0:1788575273.090997:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575273.090998:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575273.091004:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516802304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.091010:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575273.091012:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575273.091014:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575273.091016:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516802304, offset 0 00000400:00000200:0.0:1788575273.091020:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575273.091024:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575273.091027:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575273.091030:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575273.091032:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575273.091034:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575273.091036:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575273.091039:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547920:547920:256:4294967295] 192.168.202.128@tcp LPNI seq info [547843:547843:8:4294967295] 00000400:00000200:0.0:1788575273.091043:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575273.091048:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575273.091053:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575273.091056:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575273.091064:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575273.091067:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575273.091077:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575273.091079:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575273.091081:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575273.091085:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575273.091088:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575273.091089:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575273.091092:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575273.091095:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516802304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.091103:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575273.091106:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516802304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.091112:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.091114:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575273.091117:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575273.091119:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575273.091123:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575273.091124:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310633129 00000100:00000001:0.0:1788575273.091127:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310636781 00000100:00000001:0.0:1788575273.091129:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310633129 00000100:00000001:0.0:1788575273.091131:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310636781 00000100:00000001:0.0:1788575273.091132:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575273.091134:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.091136:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575273.091139:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575273/real 1788575273] req@ffff8fc4901a5880 x1875440516802304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575273.091146:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575273.091151:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575273.091153:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575273.091155:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575273.091156:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575273.091159:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575273.091162:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516802304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575273.091170:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516802304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.091176:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575273.091178:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.091179:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575273.091180:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575273.091184:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516802304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.091191:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575273.091194:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516802304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.091200:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575273.091202:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.091204:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575273.091205:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.091207:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575273.091208:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575273.091211:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575273.091214:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575273.091216:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575273.091220:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516802304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575273.091236:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802304:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575273.091240:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.091241:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575273.091244:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516801920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.091250:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575273.091252:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575273.091254:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575273.091257:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575273.091258:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575273.091259:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575273.091261:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575273.091263:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575273.091265:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575273.091267:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575273.091268:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575273.091273:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575273.091274:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.091275:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575273.091278:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516802048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.091284:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575273.091285:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575273.091286:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575273.091288:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575273.091289:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575273.091290:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575273.091292:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575273.091293:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575273.091296:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575273.091297:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575273.091299:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575273.091306:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575273.091307:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.091308:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575273.091312:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516802176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.091319:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575273.091320:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575273.091321:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575273.091323:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575273.091325:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575273.091326:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575273.091328:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575273.091329:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575273.091331:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575273.091332:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575273.091334:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575273.091336:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575273.091337:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.091339:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575273.091342:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516802304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575273.091349:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575273.091350:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575273.091351:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575273.091354:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575273.091355:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575273.091356:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575273.091357:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575273.091359:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575273.091360:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575273.091362:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575273.091363:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575273.091366:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575273.091367:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.091369:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.091372:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575273.091378:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.091380:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575273.091381:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575273.600893:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575273.600901:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575273.600905:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575273.600911:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575273.600917:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575273.600922:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.600930:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575273.600933:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575273.600937:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575274.305004:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575274.305007:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575274.305011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575274.624922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575274.624931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575274.624935:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575274.624941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575274.624951:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575274.624957:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575274.624969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575274.624972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575274.624975:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575275.648869:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575275.648880:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575275.648882:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575275.648887:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575275.648891:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575275.648896:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575275.648901:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575275.648906:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575275.648909:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575275.648934:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575275.648937:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575275.648940:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575275.648942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575275.648946:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575275.648955:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575275.648956:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575275.648958:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575275.648959:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575276.352978:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575276.352981:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575276.352983:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575276.672904:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575276.672908:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575276.672911:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575276.672915:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575276.672920:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575276.672925:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575276.672939:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575276.672941:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575276.672944:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575276.672957:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575276.672960:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575276.672961:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575276.672963:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575276.672964:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575276.672965:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575276.672967:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575276.672969:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575276.672972:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575276.928905:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310636781, at 4310636800 00000100:00000010:0.0:1788575276.928921:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575277.376992:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575277.376995:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575277.376997:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575277.696914:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575277.696921:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575277.696926:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575277.696933:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575277.696940:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575277.696947:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575277.696983:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575277.696991:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575277.696994:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.209120:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575278.209125:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575278.209128:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575278.209136:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516802432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.209148:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575278.209151:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575278.209155:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575278.209159:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575278.209169:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575278.209174:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516802432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575278.209179:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575278.209182:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575278.209184:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575278.209185:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575278.209189:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575278.209193:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575278.209199:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575278.209204:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575278.209211:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575278.209217:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516802432, portal 25 00000100:00000001:0.0:1788575278.209220:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575278.209223:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575278.209229:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516802432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.209248:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575278.209251:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575278.209254:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575278.209258:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516802432, offset 0 00000400:00000200:0.0:1788575278.209265:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575278.209273:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575278.209281:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575278.209296:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575278.209303:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575278.209309:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575278.209314:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575278.209320:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547921:547921:256:4294967295] 192.168.202.128@tcp LPNI seq info [547844:547844:8:4294967295] 00000400:00000200:0.0:1788575278.209330:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575278.209344:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575278.209353:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575278.209359:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575278.209368:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575278.209372:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575278.209398:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575278.209403:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575278.209412:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575278.209421:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575278.209429:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575278.209432:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575278.209439:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575278.209444:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516802432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.209458:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575278.209463:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516802432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.209478:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.209489:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575278.209497:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575278.209503:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575278.209507:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310641901 00000100:00000010:0.0:1788575278.209514:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575278.209523:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575278.209536:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310641901 00000100:00000010:0.0:1788575278.209543:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50120. 00000100:00000001:0.0:1788575278.209548:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575278.209555:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575278.209560:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.209565:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575278.209573:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575278/real 1788575278] req@ffff8fc4901a7800 x1875440516802432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575278.209597:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575278.209601:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575278.209604:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575278.209609:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575278.209612:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575278.209615:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575278.209620:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516802432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575278.209631:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516802432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.209647:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575278.209652:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.209658:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575278.209662:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575278.209670:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516802432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.209687:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575278.209694:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516802432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.209711:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575278.209715:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.209719:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575278.209727:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.209731:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575278.209734:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575278.209739:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575278.209747:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575278.209752:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575278.209763:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516802432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575278.209784:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516802432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575278.209792:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.209796:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575278.209801:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516802432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.209828:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575278.209831:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575278.209835:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575278.209841:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575278.209843:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575278.209847:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575278.209852:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575278.209855:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575278.209860:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575278.209870:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575278.209874:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575278.209880:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575278.209884:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.209888:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.209902:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575278.209906:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.209910:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575278.209912:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575278.209914:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575278.209920:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516802560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.209934:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575278.209936:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575278.209939:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575278.209941:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575278.209943:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575278.209947:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575278.209950:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575278.209951:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575278.209952:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575278.209954:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575278.209956:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575278.209957:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575278.209960:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575278.209962:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575278.209964:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575278.209966:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516802560, portal 10 00000100:00000001:0.0:1788575278.209967:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575278.209968:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575278.209972:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516802560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.209976:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575278.209977:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575278.209979:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575278.209987:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516802560, offset 0 00000400:00000200:0.0:1788575278.209991:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575278.209995:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575278.209998:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575278.210001:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575278.210003:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575278.210005:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575278.210008:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575278.210016:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547922:547922:256:4294967295] 192.168.202.128@tcp LPNI seq info [547845:547845:8:4294967295] 00000400:00000200:0.0:1788575278.210020:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575278.210026:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575278.210030:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575278.210032:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575278.210035:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575278.210036:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575278.210048:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575278.210049:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575278.210051:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575278.210053:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575278.210055:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575278.210056:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575278.210058:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575278.210061:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516802560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210066:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575278.210068:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516802560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210072:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.210073:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575278.210075:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575278.210077:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575278.210079:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575278.210080:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310638099 00000100:00000001:0.0:1788575278.210082:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310641901 00000100:00000001:0.0:1788575278.210083:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310638099 00000100:00000001:0.0:1788575278.210084:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310641901 00000100:00000001:0.0:1788575278.210086:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575278.210089:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.210091:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575278.210093:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575278/real 1788575278] req@ffff8fc4901a5c00 x1875440516802560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575278.210098:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575278.210099:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575278.210100:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575278.210109:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575278.210110:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575278.210114:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575278.210116:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516802560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575278.210120:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516802560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210124:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575278.210125:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210126:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575278.210127:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575278.210129:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516802560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210132:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575278.210134:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516802560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210138:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575278.210140:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210141:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575278.210142:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210143:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575278.210144:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575278.210145:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575278.210150:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575278.210151:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575278.210154:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516802560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575278.210158:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575278.210161:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575278.210163:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516802688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210166:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575278.210167:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575278.210168:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575278.210169:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575278.210170:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575278.210172:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575278.210173:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575278.210174:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575278.210175:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575278.210177:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575278.210179:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575278.210180:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575278.210182:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575278.210183:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575278.210185:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575278.210187:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516802688, portal 10 00000100:00000001:0.0:1788575278.210188:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575278.210189:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575278.210193:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516802688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210196:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575278.210200:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575278.210201:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575278.210203:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516802688, offset 0 00000400:00000200:0.0:1788575278.210205:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575278.210208:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575278.210210:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575278.210212:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575278.210213:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575278.210215:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575278.210216:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575278.210218:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547923:547923:256:4294967295] 192.168.202.128@tcp LPNI seq info [547846:547846:8:4294967295] 00000400:00000200:0.0:1788575278.210220:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575278.210224:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575278.210226:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575278.210228:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575278.210230:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575278.210232:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575278.210250:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575278.210252:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575278.210253:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575278.210256:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575278.210258:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575278.210259:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575278.210260:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575278.210262:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516802688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210267:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575278.210269:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516802688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210273:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.210277:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575278.210279:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575278.210281:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575278.210284:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575278.210285:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310638106 00000100:00000001:0.0:1788575278.210286:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310641901 00000100:00000001:0.0:1788575278.210288:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310638106 00000100:00000001:0.0:1788575278.210289:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310641901 00000100:00000001:0.0:1788575278.210290:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575278.210291:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.210292:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575278.210294:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575278/real 1788575278] req@ffff8fc4901a5180 x1875440516802688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575278.210298:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575278.210299:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575278.210300:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575278.210302:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575278.210303:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575278.210304:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575278.210306:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516802688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575278.210310:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516802688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210313:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575278.210314:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210315:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575278.210316:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575278.210318:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516802688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210321:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575278.210326:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516802688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210331:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575278.210334:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210335:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575278.210337:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210338:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575278.210341:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575278.210343:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575278.210346:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575278.210348:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575278.210352:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516802688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575278.210359:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575278.210363:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575278.210366:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516802816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210372:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575278.210374:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575278.210376:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575278.210378:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575278.210380:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575278.210382:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575278.210385:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575278.210387:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575278.210388:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575278.210390:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575278.210393:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575278.210395:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575278.210404:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575278.210406:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575278.210409:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575278.210412:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516802816, portal 4 00000100:00000001:0.0:1788575278.210414:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575278.210415:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575278.210420:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516802816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210424:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575278.210425:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575278.210427:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575278.210428:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516802816, offset 0 00000400:00000200:0.0:1788575278.210430:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575278.210433:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575278.210435:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575278.210437:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575278.210438:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575278.210440:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575278.210441:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575278.210443:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547924:547924:256:4294967295] 192.168.202.128@tcp LPNI seq info [547847:547847:8:4294967295] 00000400:00000200:0.0:1788575278.210445:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575278.210449:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575278.210452:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575278.210454:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575278.210456:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575278.210458:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575278.210464:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575278.210465:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575278.210466:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575278.210468:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575278.210470:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575278.210475:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575278.210477:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575278.210480:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516802816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210484:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575278.210486:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516802816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210490:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.210491:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575278.210493:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575278.210495:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575278.210497:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575278.210498:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310638246 00000100:00000001:0.0:1788575278.210500:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310641901 00000100:00000001:0.0:1788575278.210501:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310638246 00000100:00000001:0.0:1788575278.210502:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310641901 00000100:00000001:0.0:1788575278.210503:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575278.210504:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.210505:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575278.210507:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575278/real 1788575278] req@ffff8fc4901a7100 x1875440516802816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575278.210512:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575278.210513:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575278.210514:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575278.210517:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575278.210518:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575278.210520:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575278.210523:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516802816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575278.210530:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516802816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210541:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575278.210542:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210544:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575278.210545:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575278.210549:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516802816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210556:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575278.210559:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516802816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210564:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575278.210565:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210566:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575278.210567:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210568:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575278.210569:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575278.210570:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575278.210572:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575278.210573:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575278.210576:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516802816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575278.210580:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575278.210583:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575278.210584:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516802944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210588:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575278.210589:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575278.210594:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575278.210595:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575278.210596:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575278.210598:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575278.210600:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575278.210601:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575278.210602:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575278.210603:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575278.210605:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575278.210606:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575278.210608:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575278.210609:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575278.210611:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575278.210614:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516802944, portal 4 00000100:00000001:0.0:1788575278.210616:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575278.210617:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575278.210622:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516802944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210628:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575278.210630:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575278.210632:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575278.210634:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516802944, offset 0 00000400:00000200:0.0:1788575278.210638:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575278.210641:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575278.210644:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575278.210646:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575278.210649:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575278.210651:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575278.210653:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575278.210656:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547925:547925:256:4294967295] 192.168.202.128@tcp LPNI seq info [547848:547848:8:4294967295] 00000400:00000200:0.0:1788575278.210660:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575278.210665:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575278.210676:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575278.210679:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575278.210683:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575278.210685:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575278.210695:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575278.210697:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575278.210699:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575278.210702:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575278.210705:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575278.210707:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575278.210710:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575278.210714:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516802944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210722:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575278.210725:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516802944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210731:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.210733:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575278.210736:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575278.210739:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575278.210742:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575278.210744:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310638248 00000100:00000001:0.0:1788575278.210746:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310641901 00000100:00000001:0.0:1788575278.210748:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310638248 00000100:00000001:0.0:1788575278.210750:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310641901 00000100:00000001:0.0:1788575278.210752:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575278.210754:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.210756:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575278.210760:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575278/real 1788575278] req@ffff8fc4901a4a80 x1875440516802944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575278.210773:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575278.210775:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575278.210777:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575278.210779:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575278.210781:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575278.210783:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575278.210787:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516802944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575278.210795:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516802944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210801:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575278.210803:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210804:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575278.210806:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575278.210821:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516802944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210826:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575278.210828:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516802944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210832:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575278.210834:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210835:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575278.210836:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210837:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575278.210838:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575278.210839:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575278.210841:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575278.210842:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575278.210849:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516802944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575278.210853:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516802944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575278.210856:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210857:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575278.210859:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516802560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210862:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575278.210863:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575278.210864:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575278.210866:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575278.210867:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575278.210868:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575278.210869:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575278.210870:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575278.210872:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575278.210873:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575278.210874:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575278.210877:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575278.210878:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210879:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575278.210882:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516802688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210888:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575278.210889:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575278.210891:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575278.210893:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575278.210894:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575278.210895:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575278.210897:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575278.210898:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575278.210900:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575278.210906:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575278.210907:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575278.210910:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575278.210911:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210913:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575278.210916:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516802816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210922:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575278.210924:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575278.210925:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575278.210926:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575278.210927:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575278.210928:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575278.210929:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575278.210930:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575278.210931:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575278.210932:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575278.210933:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575278.210934:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575278.210935:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210936:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575278.210937:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516802944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575278.210941:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575278.210942:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575278.210943:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575278.210944:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575278.210944:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575278.210945:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575278.210946:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575278.210947:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575278.210948:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575278.210948:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575278.210949:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575278.210951:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575278.210952:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210956:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.210959:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575278.210959:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.210961:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575278.210962:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575278.720906:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575278.720913:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575278.720918:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575278.720923:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575278.720930:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575278.720934:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.720960:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575278.720964:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575278.720969:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575279.424966:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575279.424969:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575279.424971:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575279.744886:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575279.744894:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575279.744899:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575279.744907:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575279.744912:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575279.744916:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575279.744925:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575279.744928:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575279.744930:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575279.744947:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575279.744952:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575279.744953:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575279.744955:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575279.744957:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575279.744958:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575279.744960:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575279.744961:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575279.744963:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575280.768894:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575280.768900:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575280.768903:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575280.768925:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575280.768930:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575280.768935:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575280.768939:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575280.768941:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575280.768943:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575280.769076:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575280.769078:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575280.769079:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575280.769080:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575280.769081:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575280.769083:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575280.769084:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575280.769085:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575280.769086:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575281.473000:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575281.473002:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575281.473005:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575281.792911:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575281.792920:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575281.792925:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575281.792931:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575281.792934:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575281.792938:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575281.792952:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575281.792954:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575281.792956:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575282.048886:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310641901, at 4310641920 00000100:00000010:0.0:1788575282.048901:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575282.497006:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575282.497009:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575282.497011:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575283.329101:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575283.329107:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575283.329111:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575283.329122:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516803072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329153:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575283.329157:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575283.329163:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575283.329167:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575283.329173:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575283.329182:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516803072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575283.329191:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575283.329197:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575283.329202:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575283.329208:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575283.329214:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575283.329219:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575283.329230:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575283.329238:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575283.329247:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575283.329256:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516803072, portal 25 00000100:00000001:0.0:1788575283.329261:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575283.329264:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575283.329275:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516803072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329289:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575283.329295:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575283.329302:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575283.329306:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516803072, offset 0 00000400:00000200:0.0:1788575283.329318:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575283.329331:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575283.329341:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575283.329351:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575283.329357:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575283.329365:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575283.329372:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575283.329388:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547926:547926:256:4294967295] 192.168.202.128@tcp LPNI seq info [547849:547849:8:4294967295] 00000400:00000200:0.0:1788575283.329398:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575283.329419:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575283.329431:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575283.329439:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575283.329449:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575283.329457:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575283.329481:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575283.329485:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575283.329493:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575283.329503:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575283.329512:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575283.329517:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575283.329525:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575283.329534:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516803072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329556:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575283.329565:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516803072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329583:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.329586:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575283.329593:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575283.329601:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575283.329605:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310647021 00000100:00000010:0.0:1788575283.329612:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575283.329621:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575283.329629:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310647021 00000100:00000010:0.0:1788575283.329633:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b508a0. 00000100:00000001:0.0:1788575283.329637:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575283.329640:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575283.329646:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.329648:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575283.329654:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575283/real 1788575283] req@ffff8fc4901a4a80 x1875440516803072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575283.329665:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575283.329667:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575283.329669:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575283.329672:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575283.329674:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575283.329678:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575283.329681:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516803072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575283.329690:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516803072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329697:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575283.329699:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.329701:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575283.329703:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575283.329705:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516803072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329710:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575283.329713:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516803072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329717:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575283.329718:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.329720:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575283.329720:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.329722:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575283.329723:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575283.329725:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575283.329727:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575283.329732:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575283.329735:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516803072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575283.329741:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516803072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575283.329744:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.329745:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575283.329747:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516803072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329750:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575283.329752:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575283.329754:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575283.329756:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575283.329757:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575283.329758:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575283.329759:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575283.329761:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575283.329764:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575283.329765:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575283.329766:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575283.329769:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575283.329770:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.329771:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.329776:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575283.329777:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.329778:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575283.329779:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575283.329780:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575283.329782:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516803200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329786:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575283.329787:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575283.329789:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575283.329790:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575283.329794:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575283.329796:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516803200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575283.329803:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575283.329804:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575283.329805:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575283.329806:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575283.329826:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575283.329827:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575283.329830:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575283.329832:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575283.329834:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575283.329836:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516803200, portal 10 00000100:00000001:0.0:1788575283.329837:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575283.329838:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575283.329842:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516803200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329846:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575283.329848:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575283.329849:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575283.329850:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516803200, offset 0 00000400:00000200:0.0:1788575283.329852:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575283.329855:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575283.329866:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575283.329869:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575283.329871:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575283.329877:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575283.329879:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575283.329882:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547927:547927:256:4294967295] 192.168.202.128@tcp LPNI seq info [547850:547850:8:4294967295] 00000400:00000200:0.0:1788575283.329885:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575283.329891:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575283.329896:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575283.329904:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575283.329907:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575283.329908:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575283.329915:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575283.329917:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575283.329918:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575283.329921:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575283.329923:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575283.329924:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575283.329926:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575283.329928:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516803200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329933:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575283.329934:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516803200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329939:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.329940:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575283.329942:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575283.329944:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575283.329946:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575283.329947:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310643219 00000100:00000001:0.0:1788575283.329949:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310647021 00000100:00000001:0.0:1788575283.329950:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310643219 00000100:00000001:0.0:1788575283.329952:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310647021 00000100:00000001:0.0:1788575283.329953:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575283.329954:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.329955:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575283.329957:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575283/real 1788575283] req@ffff8fc4901a7100 x1875440516803200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575283.329968:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575283.329969:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575283.329970:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575283.329972:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575283.329973:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575283.329974:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575283.329976:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516803200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575283.329981:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516803200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329984:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575283.329985:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.329986:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575283.329987:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575283.329989:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516803200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329992:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575283.329994:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516803200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.329998:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575283.329999:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330000:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575283.330001:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330002:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575283.330003:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575283.330004:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575283.330006:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575283.330007:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575283.330012:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516803200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575283.330016:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516803200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575283.330019:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575283.330020:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516803328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330024:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575283.330025:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575283.330026:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575283.330027:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575283.330028:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575283.330029:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516803328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575283.330031:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575283.330032:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575283.330033:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575283.330034:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575283.330036:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575283.330037:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575283.330039:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575283.330040:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575283.330042:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575283.330044:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516803328, portal 10 00000100:00000001:0.0:1788575283.330045:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575283.330046:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575283.330049:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516803328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330053:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575283.330054:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575283.330055:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575283.330057:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516803328, offset 0 00000400:00000200:0.0:1788575283.330062:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575283.330064:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575283.330066:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575283.330068:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575283.330069:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575283.330071:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575283.330072:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575283.330074:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547928:547928:256:4294967295] 192.168.202.128@tcp LPNI seq info [547851:547851:8:4294967295] 00000400:00000200:0.0:1788575283.330090:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575283.330095:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575283.330098:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575283.330100:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575283.330102:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575283.330104:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575283.330110:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575283.330111:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575283.330113:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575283.330115:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575283.330117:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575283.330118:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575283.330119:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575283.330121:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516803328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330126:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575283.330128:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516803328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330131:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.330132:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575283.330134:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575283.330136:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575283.330138:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575283.330142:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310643226 00000100:00000001:0.0:1788575283.330144:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310647021 00000100:00000001:0.0:1788575283.330145:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310643226 00000100:00000001:0.0:1788575283.330146:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310647021 00000100:00000001:0.0:1788575283.330148:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575283.330149:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.330150:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575283.330152:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575283/real 1788575283] req@ffff8fc4901a5180 x1875440516803328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575283.330156:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575283.330158:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575283.330159:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575283.330160:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575283.330161:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575283.330162:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575283.330164:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516803328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575283.330169:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516803328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330172:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575283.330173:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330174:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575283.330175:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575283.330177:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516803328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330180:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575283.330182:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516803328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330186:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575283.330190:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330191:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575283.330192:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330193:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575283.330194:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575283.330196:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575283.330198:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575283.330199:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575283.330201:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516803328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575283.330205:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516803328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575283.330207:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575283.330209:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516803456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330213:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575283.330213:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575283.330215:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575283.330216:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575283.330217:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575283.330218:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516803456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575283.330220:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575283.330221:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575283.330222:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575283.330223:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575283.330224:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575283.330226:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575283.330228:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575283.330229:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575283.330231:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575283.330233:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516803456, portal 4 00000100:00000001:0.0:1788575283.330236:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575283.330237:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575283.330242:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516803456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330248:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575283.330250:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575283.330252:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575283.330254:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516803456, offset 0 00000400:00000200:0.0:1788575283.330258:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575283.330262:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575283.330265:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575283.330268:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575283.330270:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575283.330272:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575283.330275:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575283.330277:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547929:547929:256:4294967295] 192.168.202.128@tcp LPNI seq info [547852:547852:8:4294967295] 00000400:00000200:0.0:1788575283.330281:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575283.330287:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575283.330290:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575283.330292:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575283.330295:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575283.330296:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575283.330302:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575283.330303:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575283.330305:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575283.330307:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575283.330309:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575283.330310:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575283.330311:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575283.330314:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516803456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330327:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575283.330329:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516803456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330333:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.330333:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575283.330336:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575283.330338:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575283.330340:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575283.330341:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310643366 00000100:00000001:0.0:1788575283.330343:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310647021 00000100:00000001:0.0:1788575283.330344:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310643366 00000100:00000001:0.0:1788575283.330345:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310647021 00000100:00000001:0.0:1788575283.330346:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575283.330347:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.330348:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575283.330350:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575283/real 1788575283] req@ffff8fc4901a5c00 x1875440516803456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575283.330354:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575283.330355:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575283.330356:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575283.330358:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575283.330358:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575283.330360:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575283.330362:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516803456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575283.330366:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516803456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330369:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575283.330373:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330374:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575283.330375:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575283.330377:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516803456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330381:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575283.330383:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516803456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330386:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575283.330388:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330389:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575283.330390:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330391:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575283.330392:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575283.330393:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575283.330395:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575283.330396:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575283.330398:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516803456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575283.330402:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516803456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575283.330404:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575283.330406:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516803584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330409:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575283.330410:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575283.330412:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575283.330413:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575283.330414:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575283.330418:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516803584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575283.330419:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575283.330421:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575283.330422:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575283.330423:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575283.330425:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575283.330426:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575283.330428:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575283.330429:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575283.330431:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575283.330433:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516803584, portal 4 00000100:00000001:0.0:1788575283.330434:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575283.330435:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575283.330438:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516803584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330442:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575283.330443:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575283.330444:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575283.330446:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516803584, offset 0 00000400:00000200:0.0:1788575283.330448:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575283.330449:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575283.330451:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575283.330453:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575283.330454:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575283.330455:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575283.330457:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575283.330459:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547930:547930:256:4294967295] 192.168.202.128@tcp LPNI seq info [547853:547853:8:4294967295] 00000400:00000200:0.0:1788575283.330461:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575283.330464:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575283.330467:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575283.330468:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575283.330473:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575283.330475:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575283.330487:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575283.330489:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575283.330492:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575283.330495:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575283.330499:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575283.330500:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575283.330503:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575283.330506:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516803584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330514:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575283.330517:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516803584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330523:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.330524:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575283.330527:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575283.330528:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575283.330530:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575283.330532:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310643368 00000100:00000001:0.0:1788575283.330533:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310647021 00000100:00000001:0.0:1788575283.330534:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310643368 00000100:00000001:0.0:1788575283.330536:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310647021 00000100:00000001:0.0:1788575283.330537:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575283.330538:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.330540:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575283.330542:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575283/real 1788575283] req@ffff8fc4901a7800 x1875440516803584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575283.330546:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575283.330550:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575283.330551:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575283.330552:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575283.330553:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575283.330555:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575283.330557:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516803584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575283.330562:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516803584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330565:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575283.330566:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330567:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575283.330568:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575283.330570:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516803584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330573:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575283.330575:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516803584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330579:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575283.330580:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330581:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575283.330581:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330582:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575283.330583:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575283.330585:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575283.330587:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575283.330588:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575283.330590:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516803584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575283.330600:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516803584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575283.330602:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330603:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575283.330605:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516803200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330608:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575283.330609:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575283.330611:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575283.330612:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575283.330613:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575283.330614:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575283.330615:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575283.330616:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575283.330618:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575283.330619:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575283.330620:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575283.330622:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575283.330623:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330624:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575283.330625:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516803328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330629:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575283.330629:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575283.330630:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575283.330631:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575283.330632:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575283.330633:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575283.330634:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575283.330635:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575283.330636:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575283.330636:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575283.330637:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575283.330642:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575283.330643:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330644:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575283.330646:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516803456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330650:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575283.330651:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575283.330652:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575283.330653:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575283.330654:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575283.330655:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575283.330656:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575283.330658:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575283.330659:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575283.330660:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575283.330661:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575283.330663:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575283.330664:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330666:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575283.330669:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516803584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575283.330675:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575283.330676:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575283.330677:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575283.330680:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575283.330681:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575283.330682:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575283.330684:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575283.330685:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575283.330688:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575283.330689:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575283.330690:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575283.330692:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575283.330693:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330695:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.330698:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575283.330703:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.330705:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575283.330706:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.841141:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575283.841145:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575283.841146:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575283.841150:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575283.841152:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575283.841156:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.841160:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575283.841163:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.841168:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575283.841231:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575283.841234:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575283.841236:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575283.841238:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575283.841241:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575283.841243:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.841245:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575283.841247:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575283.841249:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575284.545007:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575284.545010:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575284.545012:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575284.864925:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575284.864931:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575284.864936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575284.864942:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575284.864950:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575284.864956:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575284.864980:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575284.864983:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575284.864987:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575285.888925:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575285.888931:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575285.888934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575285.888939:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575285.888943:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575285.888962:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575285.888978:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575285.888981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575285.889006:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575286.592952:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575286.592955:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575286.592957:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575286.912887:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575286.912894:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575286.912907:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575286.912914:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575286.912922:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575286.912928:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575286.912939:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575286.912945:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575286.912955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575287.168911:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310647021, at 4310647040 00000100:00000010:0.0:1788575287.168927:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575287.616984:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575287.616990:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575287.616993:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575287.936984:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575287.936996:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575287.937004:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575287.937012:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575287.937026:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575287.937032:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575287.937038:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575287.937044:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575287.937055:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575288.448902:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575288.448926:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575288.448930:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575288.448935:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575288.448943:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575288.448953:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15975 00000400:00000200:0.0:1788575288.448990:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575288.448998:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575288.449001:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575288.449004:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449007:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575288.449010:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449013:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575288.449018:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575288.449020:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575288.449023:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575288.449027:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575288.449029:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575288.449032:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575288.449035:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449037:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575288.449039:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449041:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575288.449044:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575288.449048:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449051:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575288.449054:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575288.449058:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575288.449062:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449064:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575288.449067:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575288.449074:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575288.449078:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575288.449083:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575288.449085:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449088:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575288.449091:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575288.449097:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575288.449112:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575288.449119:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575288.449131:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516803712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575288.449142:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575288.449148:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575288.449153:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516803712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575288.449177:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575288.449180:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575288.449184:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575288.449186:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575288.449188:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575288.449191:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575288.449194:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15975 00000400:00000200:0.0:1788575288.449199:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575288.449204:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575288.449208:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575288.449215:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449219:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575288.449223:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449225:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575288.449227:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575288.449229:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575288.449233:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575288.449236:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575288.449240:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575288.449244:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575288.449249:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449261:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575288.449271:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449275:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575288.449279:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575288.449286:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449289:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575288.449293:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575288.449301:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575288.449305:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449310:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575288.449315:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575288.449321:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449326:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575288.449332:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575288.449336:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575288.449341:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575288.449343:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449347:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575288.449349:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575288.449353:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575288.449359:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575288.449363:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575288.449369:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516803840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575288.449378:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575288.449380:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575288.449383:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516803840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575288.449393:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575288.449397:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575288.449403:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575288.449408:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575288.449410:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575288.449414:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575288.449418:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15975 00000400:00000200:0.0:1788575288.449423:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575288.449427:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575288.449429:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575288.449434:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449437:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575288.449441:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449444:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575288.449447:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575288.449449:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575288.449451:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575288.449453:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575288.449457:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575288.449461:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575288.449466:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449469:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575288.449474:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449475:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575288.449476:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575288.449479:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449482:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575288.449485:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575288.449488:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575288.449493:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449495:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575288.449496:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575288.449498:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449501:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575288.449506:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575288.449509:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575288.449511:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575288.449512:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449516:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575288.449518:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575288.449519:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575288.449522:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575288.449526:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575288.449532:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516803968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575288.449541:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575288.449544:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575288.449548:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516803968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575288.449557:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575288.449560:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575288.449564:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575288.449565:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575288.449567:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575288.449572:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575288.449574:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15975 00000400:00000200:0.0:1788575288.449576:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575288.449578:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575288.449580:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575288.449582:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449583:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575288.449584:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449586:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575288.449589:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575288.449597:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575288.449609:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575288.449612:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575288.449614:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575288.449618:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575288.449623:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449627:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575288.449630:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449632:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575288.449634:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575288.449635:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449636:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575288.449637:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575288.449639:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575288.449642:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449645:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575288.449648:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575288.449650:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449652:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575288.449655:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575288.449659:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575288.449661:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575288.449663:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449667:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575288.449669:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575288.449671:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575288.449676:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575288.449681:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575288.449688:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516804096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575288.449697:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575288.449700:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575288.449706:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516804096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575288.449716:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575288.449719:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575288.449723:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575288.449726:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575288.449727:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575288.449729:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575288.449732:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15975 00000400:00000200:0.0:1788575288.449737:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575288.449739:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575288.449740:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575288.449742:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449744:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575288.449748:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449750:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575288.449751:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575288.449752:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575288.449753:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575288.449755:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575288.449758:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575288.449761:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575288.449763:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449767:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575288.449769:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449773:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575288.449776:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575288.449778:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575288.449783:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575288.449789:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575288.449792:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575288.449794:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449796:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575288.449798:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575288.449800:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449803:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575288.449830:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575288.449832:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575288.449834:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575288.449835:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575288.449838:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575288.449840:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575288.449843:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575288.449849:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575288.449853:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575288.449859:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516804224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575288.449868:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575288.449873:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575288.449881:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516804224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575288.449898:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575288.449906:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15986) 00000100:00000001:0.0:1788575288.960926:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575288.960938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575288.960946:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575288.960955:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575288.960967:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575288.960972:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575288.960994:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575288.961001:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575288.961007:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575289.665056:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575289.665078:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575289.665081:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575289.984969:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575289.984980:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575289.984983:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575289.984992:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575289.985005:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575289.985011:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575289.985017:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575289.985023:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575289.985026:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575291.008992:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575291.009000:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575291.009005:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575291.009010:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575291.009023:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575291.009038:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575291.009045:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575291.009046:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575291.009048:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575291.713048:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575291.713053:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575291.713056:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575292.032976:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575292.032988:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575292.032993:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575292.033008:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575292.033017:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575292.033025:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575292.033035:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575292.033041:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575292.033049:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575292.288941:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310652141, at 4310652160 00000100:00000010:0.0:1788575292.288964:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf900. 00000020:00000001:0.0:1788575292.737057:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575292.737060:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575292.737063:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788575294.785073:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575294.785076:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575294.785079:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575295.104981:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575295.104985:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575295.104990:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575295.104995:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575295.105001:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575295.105006:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575295.105027:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575295.105029:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575295.105035:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575296.129010:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575296.129017:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575296.129021:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575296.129027:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575296.129033:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575296.129038:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575296.129041:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575296.129044:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575296.129048:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575296.832987:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575296.832990:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575296.832993:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575297.152954:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575297.152959:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575297.152963:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575297.152969:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575297.152976:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575297.152982:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575297.152990:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575297.152994:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575297.152998:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575297.408905:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310657261, at 4310657280 00000100:00000010:0.0:1788575297.408924:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf300. 00000020:00000001:0.0:1788575297.857000:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575297.857004:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575297.857026:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575298.176978:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575298.176987:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575298.176991:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575298.177000:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575298.177007:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575298.177014:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575298.177020:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575298.177024:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575298.177027:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575298.688965:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575298.688981:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575298.688984:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575298.688990:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575298.688996:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575298.689007:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15986 00000400:00000200:0.0:1788575298.689019:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575298.689027:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575298.689031:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575298.689034:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689046:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575298.689049:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689052:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575298.689056:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575298.689059:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575298.689062:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575298.689065:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575298.689068:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575298.689071:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575298.689074:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689076:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575298.689078:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689106:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575298.689109:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575298.689113:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689116:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575298.689119:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575298.689130:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575298.689134:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.689136:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575298.689139:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575298.689149:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575298.689153:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575298.689157:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575298.689159:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.689163:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575298.689167:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575298.689173:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575298.689181:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575298.689190:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575298.689201:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516804992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575298.689220:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575298.689225:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575298.689230:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516804992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575298.689252:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575298.689255:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575298.689259:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575298.689261:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575298.689262:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575298.689265:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575298.689268:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15986 00000400:00000200:0.0:1788575298.689281:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575298.689288:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575298.689292:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575298.689310:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689316:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575298.689322:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689328:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575298.689333:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575298.689335:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575298.689339:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575298.689348:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575298.689352:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575298.689358:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575298.689366:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689370:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575298.689375:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689378:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575298.689382:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575298.689389:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689395:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575298.689400:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575298.689408:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575298.689414:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.689419:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575298.689425:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575298.689431:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.689434:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575298.689439:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575298.689442:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575298.689446:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575298.689455:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.689459:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575298.689462:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575298.689467:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575298.689474:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575298.689482:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575298.689491:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516805120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575298.689508:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575298.689512:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575298.689518:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516805120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575298.689536:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575298.689541:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575298.689545:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575298.689547:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575298.689549:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575298.689552:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575298.689557:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15986 00000400:00000200:0.0:1788575298.689566:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575298.689573:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575298.689579:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575298.689585:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689604:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575298.689610:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689616:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575298.689622:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575298.689632:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575298.689636:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575298.689640:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575298.689644:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575298.689655:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575298.689663:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689668:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575298.689673:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689678:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575298.689683:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575298.689691:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689696:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575298.689700:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575298.689705:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575298.689709:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.689714:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575298.689718:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575298.689724:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.689728:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575298.689735:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575298.689739:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575298.689744:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575298.689747:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.689753:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575298.689756:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575298.689760:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575298.689767:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575298.689784:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575298.689796:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516805248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575298.689851:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575298.689859:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575298.689870:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516805248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575298.689893:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575298.689903:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575298.689910:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575298.689915:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575298.689920:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575298.689928:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575298.689934:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15986 00000400:00000200:0.0:1788575298.689942:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575298.689947:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575298.689951:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575298.689956:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689961:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575298.689966:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.689971:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575298.689976:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575298.689979:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575298.689984:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575298.689989:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575298.689993:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575298.689998:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575298.690005:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.690011:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575298.690015:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.690020:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575298.690026:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575298.690033:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575298.690038:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575298.690042:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575298.690046:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575298.690050:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.690052:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575298.690060:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575298.690066:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.690070:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575298.690074:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575298.690079:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575298.690085:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575298.690089:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.690093:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575298.690097:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575298.690102:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575298.690110:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575298.690117:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575298.690128:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516805376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575298.690141:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575298.690146:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575298.690156:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516805376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575298.690177:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575298.690182:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575298.690191:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575298.690194:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575298.690197:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575298.690202:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575298.690207:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15986 00000400:00000200:0.0:1788575298.690211:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575298.690216:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575298.690220:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575298.690227:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.690233:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575298.690238:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.690250:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575298.690256:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575298.690260:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575298.690265:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575298.690269:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575298.690276:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575298.690282:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575298.690292:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.690297:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575298.690303:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575298.690306:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575298.690312:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575298.690317:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575298.690324:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575298.690328:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575298.690333:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575298.690338:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.690341:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575298.690344:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575298.690347:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.690351:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575298.690359:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575298.690364:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575298.690369:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575298.690372:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575298.690376:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575298.690380:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575298.690385:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575298.690392:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575298.690399:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575298.690409:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516805504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575298.690443:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575298.690448:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575298.690456:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516805504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575298.690479:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575298.690487:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (15996) 00000020:00000001:0.0:1788575299.905076:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575299.905082:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575299.905086:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575300.224941:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575300.224951:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575300.224955:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575300.224960:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575300.224968:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575300.224972:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575300.224984:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575300.224989:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575300.225001:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575300.225038:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575300.225041:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575300.225043:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575300.225045:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575300.225048:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575300.225051:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575300.225052:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575300.225054:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575300.225056:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788575300.737155:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575300.737172:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575300.737178:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575300.737181:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575301.248926:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575301.248934:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575301.248939:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575301.248948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575301.248971:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575301.248976:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575301.249000:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575301.249004:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575301.249006:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575301.952988:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575301.952990:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575301.952993:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575302.272898:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575302.272906:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575302.272911:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575302.272920:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575302.272927:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575302.272932:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575302.272951:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575302.272952:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575302.272955:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575302.528915:0:265885:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310662381, at 4310662400 00000100:00000010:0.0:1788575302.528930:0:265885:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf840. 00000020:00000001:0.0:1788575302.977033:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575302.977038:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575302.977041:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575303.296943:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575303.296952:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575303.296956:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575303.296962:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575303.296970:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575303.296977:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575303.296999:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575303.297011:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575303.297034:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575303.808890:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575303.808898:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575303.808901:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575303.808904:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575303.808909:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575303.808927:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 15991 00000400:00000200:0.0:1788575303.808936:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575303.808940:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575303.808943:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575303.808945:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.808948:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575303.808950:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.808952:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575303.808955:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575303.808956:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575303.808958:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575303.808961:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575303.808962:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575303.808964:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575303.808966:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.808967:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575303.808969:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.808970:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575303.808972:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575303.808974:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575303.808976:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575303.808978:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575303.808980:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575303.808982:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.808983:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575303.808985:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575303.808989:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575303.808991:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575303.808994:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575303.808995:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.808998:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575303.808999:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575303.809006:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575303.809012:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575303.809018:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575303.809025:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516805632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575303.809032:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575303.809035:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575303.809039:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516805632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575303.809057:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575303.809060:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575303.809062:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575303.809063:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575303.809065:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575303.809066:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575303.809068:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15991 00000400:00000200:0.0:1788575303.809071:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575303.809073:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575303.809076:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575303.809079:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809082:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575303.809084:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809087:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575303.809096:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575303.809097:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575303.809099:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575303.809102:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575303.809104:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575303.809107:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575303.809114:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809124:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575303.809127:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809131:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575303.809135:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575303.809142:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809148:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575303.809150:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575303.809156:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575303.809160:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.809164:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575303.809169:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575303.809175:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.809180:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575303.809186:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575303.809192:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575303.809198:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575303.809202:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.809206:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575303.809210:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575303.809213:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575303.809221:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575303.809228:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575303.809238:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516805760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575303.809254:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575303.809259:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575303.809266:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516805760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575303.809278:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575303.809284:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575303.809291:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575303.809302:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575303.809306:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575303.809312:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575303.809319:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15991 00000400:00000200:0.0:1788575303.809327:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575303.809336:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575303.809341:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575303.809347:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809351:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575303.809357:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809361:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575303.809363:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575303.809365:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575303.809370:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575303.809375:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575303.809378:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575303.809382:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575303.809390:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809395:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575303.809401:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809407:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575303.809413:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575303.809422:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809444:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575303.809450:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575303.809456:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575303.809460:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.809464:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575303.809469:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575303.809475:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.809484:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575303.809490:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575303.809496:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575303.809501:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575303.809505:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.809509:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575303.809513:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575303.809518:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575303.809525:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575303.809533:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575303.809543:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516805888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575303.809558:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575303.809563:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575303.809571:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516805888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575303.809585:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575303.809591:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575303.809601:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575303.809605:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575303.809609:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575303.809614:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575303.809621:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15991 00000400:00000200:0.0:1788575303.809629:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575303.809636:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575303.809641:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575303.809648:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809653:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575303.809657:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809663:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575303.809666:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575303.809678:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575303.809683:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575303.809688:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575303.809693:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575303.809697:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575303.809702:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809707:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575303.809710:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809713:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575303.809719:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575303.809726:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809731:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575303.809737:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575303.809743:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575303.809747:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.809749:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575303.809751:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575303.809754:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.809756:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575303.809760:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575303.809762:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575303.809765:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575303.809766:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.809768:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575303.809769:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575303.809772:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575303.809788:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575303.809797:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575303.809804:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516806016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575303.809841:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575303.809843:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575303.809855:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516806016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575303.809868:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575303.809871:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575303.809875:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575303.809877:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575303.809879:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575303.809885:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575303.809891:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 15991 00000400:00000200:0.0:1788575303.809902:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575303.809907:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575303.809912:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575303.809918:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809923:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575303.809929:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809933:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575303.809939:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575303.809941:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575303.809945:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575303.809949:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575303.809953:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575303.809960:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575303.809967:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809971:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575303.809972:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809974:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575303.809980:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575303.809986:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575303.809989:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575303.810000:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575303.810006:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575303.810010:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.810015:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575303.810019:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575303.810024:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.810028:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575303.810035:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575303.810041:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575303.810048:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575303.810050:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575303.810052:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575303.810055:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575303.810059:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575303.810067:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575303.810071:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575303.810077:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516806144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575303.810086:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575303.810088:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575303.810091:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516806144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575303.810099:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575303.810103:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16001) 00000100:00000001:0.0:1788575304.320916:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575304.320925:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575304.320930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575304.320939:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575304.320947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575304.320953:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575304.320967:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575304.320971:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575304.320975:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575305.025010:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575305.025029:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575305.025032:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575305.344976:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575305.344981:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575305.344983:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575305.344986:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575305.344989:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575305.344992:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575305.344994:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575305.344996:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575305.344999:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575305.345049:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575305.345051:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575305.345053:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575305.345055:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575305.345057:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575305.345059:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575305.345061:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575305.345071:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575305.345075:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575306.368905:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575306.368914:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575306.368919:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575306.368925:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575306.368932:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575306.368938:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575306.368954:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575306.368960:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575306.368965:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575306.368984:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575306.368989:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575306.368991:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575306.368993:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575306.368996:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575306.368999:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575306.369002:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575306.369005:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575306.369008:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575307.073008:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575307.073010:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575307.073013:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575307.392919:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575307.392928:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575307.392933:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575307.392943:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575307.392951:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575307.392954:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575307.392959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575307.392963:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575307.392968:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575307.392985:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575307.392988:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575307.392990:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575307.392995:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575307.393000:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575307.393003:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575307.393006:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575307.393007:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575307.393009:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575307.649010:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310667501, at 4310667520 00000100:00000010:0.0:1788575307.649024:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfc00. 00000020:00000001:0.0:1788575308.097017:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575308.097021:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575308.097025:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575308.416915:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575308.416925:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575308.416930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575308.416939:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575308.416947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575308.416954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575308.416972:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575308.416978:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575308.416985:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575309.440963:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575309.440974:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575309.440999:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575309.441012:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575309.441022:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575309.441027:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575309.441041:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575309.441047:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575309.441052:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575310.145118:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575310.145123:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575310.145126:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575310.464913:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575310.464920:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575310.464923:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575310.464930:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575310.464935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575310.464939:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575310.464942:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575310.464946:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575310.464950:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575311.488961:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575311.488967:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575311.488975:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575311.488993:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575311.489011:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575311.489015:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575311.489018:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575311.489019:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575311.489023:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575312.193014:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575312.193017:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575312.193020:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575312.512948:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575312.512957:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575312.512973:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575312.512981:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575312.512998:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575312.513006:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575312.513013:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575312.513031:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575312.513035:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575312.513065:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575312.513073:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575312.513082:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575312.513089:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575312.513102:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575312.513111:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575312.513116:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575312.513118:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575312.513122:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575312.768911:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310672621, at 4310672640 00000100:00000010:0.0:1788575312.768929:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf7e0. 00000020:00000001:0.0:1788575313.217066:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575313.217072:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575313.217075:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575313.536966:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575313.536972:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575313.536975:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575313.536983:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575313.536992:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575313.536996:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575313.537012:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575313.537018:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575313.537022:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.049215:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575314.049223:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575314.049227:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575314.049239:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516806912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.049259:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575314.049262:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575314.049266:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575314.049270:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575314.049277:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575314.049285:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516806912:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575314.049309:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575314.049315:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575314.049319:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575314.049326:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575314.049333:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575314.049337:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575314.049347:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575314.049352:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575314.049359:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575314.049367:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516806912, portal 25 00000100:00000001:0.0:1788575314.049381:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575314.049384:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575314.049394:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516806912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.049414:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575314.049421:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575314.049425:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575314.049429:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516806912, offset 0 00000400:00000200:0.0:1788575314.049436:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575314.049445:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575314.049457:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575314.049463:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575314.049467:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575314.049472:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575314.049478:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575314.049484:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547956:547956:256:4294967295] 192.168.202.128@tcp LPNI seq info [547879:547879:8:4294967295] 00000400:00000200:0.0:1788575314.049492:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575314.049509:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575314.049522:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575314.049530:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575314.049539:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575314.049556:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575314.049581:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575314.049585:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575314.049591:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575314.049599:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575314.049605:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575314.049609:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575314.049615:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575314.049624:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516806912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.049648:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575314.049658:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516806912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.049678:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.049684:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575314.049693:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575314.049701:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575314.049713:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310677741 00000100:00000010:0.0:1788575314.049720:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575314.049729:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575314.049735:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310677741 00000100:00000010:0.0:1788575314.049740:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50d80. 00000100:00000001:0.0:1788575314.049751:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575314.049755:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575314.049764:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.049770:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575314.049780:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575314/real 1788575314] req@ffff8fc4901a5880 x1875440516806912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575314.049804:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575314.049828:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575314.049837:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575314.049841:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575314.049845:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575314.049851:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575314.049861:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516806912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575314.049893:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516806912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.049912:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575314.049916:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.049921:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575314.049926:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575314.049935:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516806912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.049947:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575314.049952:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516806912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.049958:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575314.049963:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.049964:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575314.049966:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.049972:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575314.049974:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575314.049979:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575314.049984:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575314.049990:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575314.050000:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516806912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575314.050021:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516806912:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575314.050040:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.050043:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575314.050047:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516806912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050052:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575314.050054:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575314.050056:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575314.050059:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575314.050061:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575314.050062:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575314.050064:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575314.050067:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575314.050070:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575314.050072:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575314.050074:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575314.050079:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575314.050080:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.050082:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.050089:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575314.050091:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.050093:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575314.050095:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575314.050097:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575314.050117:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516807040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050128:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575314.050130:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575314.050133:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575314.050137:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575314.050139:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575314.050143:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807040:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575314.050146:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575314.050148:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575314.050150:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575314.050158:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575314.050161:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575314.050163:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575314.050167:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575314.050170:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575314.050174:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575314.050177:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516807040, portal 10 00000100:00000001:0.0:1788575314.050179:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575314.050180:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575314.050187:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516807040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050193:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575314.050195:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575314.050197:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575314.050200:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516807040, offset 0 00000400:00000200:0.0:1788575314.050203:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575314.050208:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575314.050211:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575314.050215:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575314.050218:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575314.050220:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575314.050222:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575314.050225:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547957:547957:256:4294967295] 192.168.202.128@tcp LPNI seq info [547880:547880:8:4294967295] 00000400:00000200:0.0:1788575314.050229:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575314.050236:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575314.050240:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575314.050244:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575314.050248:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575314.050251:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575314.050260:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575314.050262:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575314.050272:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575314.050275:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575314.050279:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575314.050281:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575314.050284:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575314.050287:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516807040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050295:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575314.050298:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516807040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050304:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.050305:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575314.050309:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575314.050312:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575314.050315:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575314.050317:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310673939 00000100:00000001:0.0:1788575314.050319:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310677741 00000100:00000001:0.0:1788575314.050321:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310673939 00000100:00000001:0.0:1788575314.050324:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310677741 00000100:00000001:0.0:1788575314.050326:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575314.050328:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.050329:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575314.050333:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575314/real 1788575314] req@ffff8fc4901a4700 x1875440516807040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575314.050340:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575314.050341:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575314.050343:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575314.050346:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575314.050347:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575314.050349:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575314.050358:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516807040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575314.050366:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516807040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050372:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575314.050374:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.050375:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575314.050377:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575314.050380:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516807040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050386:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575314.050390:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516807040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050396:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575314.050398:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.050400:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575314.050401:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.050403:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575314.050404:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575314.050407:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575314.050410:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575314.050412:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575314.050416:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516807040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575314.050424:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807040:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575314.050428:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575314.050431:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516807168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050450:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575314.050451:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575314.050453:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575314.050455:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575314.050457:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575314.050460:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807168:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575314.050463:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575314.050465:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575314.050467:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575314.050468:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575314.050472:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575314.050474:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575314.050477:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575314.050480:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575314.050484:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575314.050487:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516807168, portal 10 00000100:00000001:0.0:1788575314.050489:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575314.050490:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575314.050496:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516807168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050502:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575314.050504:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575314.050507:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575314.050509:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516807168, offset 0 00000400:00000200:0.0:1788575314.050513:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575314.050527:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575314.050530:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575314.050535:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575314.050537:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575314.050538:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575314.050540:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575314.050547:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547958:547958:256:4294967295] 192.168.202.128@tcp LPNI seq info [547881:547881:8:4294967295] 00000400:00000200:0.0:1788575314.050550:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575314.050555:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575314.050558:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575314.050561:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575314.050563:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575314.050565:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575314.050573:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575314.050574:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575314.050576:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575314.050579:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575314.050581:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575314.050582:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575314.050584:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575314.050587:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516807168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050592:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575314.050594:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516807168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050599:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.050600:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575314.050602:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575314.050604:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575314.050607:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575314.050608:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310673946 00000100:00000001:0.0:1788575314.050610:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310677741 00000100:00000001:0.0:1788575314.050611:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310673946 00000100:00000001:0.0:1788575314.050613:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310677741 00000100:00000001:0.0:1788575314.050614:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575314.050619:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.050620:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575314.050623:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575314/real 1788575314] req@ffff8fc4901a7480 x1875440516807168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575314.050628:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575314.050629:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575314.050631:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575314.050632:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575314.050633:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575314.050635:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575314.050638:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516807168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575314.050648:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516807168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050654:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575314.050656:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.050658:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575314.050659:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575314.050663:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516807168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050670:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575314.050675:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516807168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050683:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575314.050685:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.050687:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575314.050688:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.050690:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575314.050693:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575314.050695:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575314.050702:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575314.050704:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575314.050707:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516807168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575314.050713:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807168:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575314.050715:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575314.050717:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516807296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050721:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575314.050723:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575314.050724:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575314.050726:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575314.050727:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575314.050728:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807296:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575314.050731:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575314.050732:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575314.050733:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575314.050735:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575314.050737:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575314.050738:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575314.050741:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575314.050743:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575314.050745:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575314.050747:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516807296, portal 4 00000100:00000001:0.0:1788575314.050749:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575314.050750:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575314.050754:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516807296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050759:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575314.050765:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575314.050768:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575314.050770:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516807296, offset 0 00000400:00000200:0.0:1788575314.050775:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575314.050779:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575314.050782:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575314.050785:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575314.050788:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575314.050790:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575314.050793:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575314.050796:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547959:547959:256:4294967295] 192.168.202.128@tcp LPNI seq info [547882:547882:8:4294967295] 00000400:00000200:0.0:1788575314.050801:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575314.050826:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575314.050832:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575314.050835:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575314.050839:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575314.050842:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575314.050859:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575314.050861:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575314.050864:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575314.050868:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575314.050872:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575314.050874:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575314.050877:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575314.050881:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516807296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050890:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575314.050893:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516807296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050900:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.050901:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575314.050911:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575314.050914:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575314.050918:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575314.050920:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310674087 00000100:00000001:0.0:1788575314.050922:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310677741 00000100:00000001:0.0:1788575314.050925:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310674087 00000100:00000001:0.0:1788575314.050927:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310677741 00000100:00000001:0.0:1788575314.050929:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575314.050931:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.050932:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575314.050935:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575314/real 1788575314] req@ffff8fc4901a6300 x1875440516807296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575314.050941:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575314.050942:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575314.050944:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575314.050945:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575314.050946:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575314.050948:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575314.050950:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516807296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575314.050955:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516807296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050959:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575314.050960:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.050961:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575314.050962:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575314.050964:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516807296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050969:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575314.050972:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516807296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.050981:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575314.050983:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.050984:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575314.050985:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.050986:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575314.050987:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575314.050989:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575314.050991:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575314.050992:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575314.050995:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516807296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575314.051000:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807296:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575314.051002:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575314.051004:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516807424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.051008:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575314.051009:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575314.051011:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575314.051012:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575314.051013:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575314.051015:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807424:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575314.051017:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575314.051018:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575314.051019:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575314.051020:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575314.051023:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575314.051024:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575314.051026:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575314.051031:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575314.051033:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575314.051035:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516807424, portal 4 00000100:00000001:0.0:1788575314.051037:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575314.051038:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575314.051042:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516807424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.051046:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575314.051048:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575314.051049:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575314.051051:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516807424, offset 0 00000400:00000200:0.0:1788575314.051054:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575314.051058:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575314.051061:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575314.051064:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575314.051067:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575314.051069:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575314.051071:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575314.051074:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547960:547960:256:4294967295] 192.168.202.128@tcp LPNI seq info [547883:547883:8:4294967295] 00000400:00000200:0.0:1788575314.051079:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575314.051085:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575314.051090:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575314.051094:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575314.051098:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575314.051114:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575314.051125:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575314.051127:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575314.051131:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575314.051135:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575314.051138:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575314.051140:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575314.051149:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575314.051153:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516807424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.051162:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575314.051166:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516807424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575339 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.051172:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.051174:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575314.051178:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575314.051181:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575314.051185:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575314.051188:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310674089 00000100:00000001:0.0:1788575314.051191:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310677741 00000100:00000001:0.0:1788575314.051193:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310674089 00000100:00000001:0.0:1788575314.051195:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310677741 00000100:00000001:0.0:1788575314.051197:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575314.051199:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.051200:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575314.051203:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575314/real 1788575314] req@ffff8fc4901a4e00 x1875440516807424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575314.051209:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575314.051210:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575314.051211:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575314.051213:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575314.051214:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575314.051216:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575314.051218:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516807424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575314.051223:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516807424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.051239:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575314.051241:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.051243:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575314.051245:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575314.051249:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516807424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.051256:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575314.051260:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516807424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.051268:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575314.051270:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.051272:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575314.051273:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.051275:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575314.051277:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575314.051280:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575314.051283:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575314.051286:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575314.051291:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516807424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575314.051300:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807424:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575314.051304:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.051306:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575314.051310:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516807040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.051318:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575314.051319:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575314.051327:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575314.051330:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575314.051332:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575314.051333:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575314.051336:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575314.051337:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575314.051340:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575314.051342:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575314.051343:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575314.051347:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575314.051349:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.051351:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575314.051355:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516807168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.051363:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575314.051365:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575314.051366:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575314.051369:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575314.051370:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575314.051372:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575314.051374:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575314.051375:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575314.051377:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575314.051378:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575314.051379:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575314.051381:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575314.051382:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.051383:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575314.051386:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516807296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.051390:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575314.051391:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575314.051392:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575314.051393:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575314.051394:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575314.051395:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575314.051400:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575314.051401:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575314.051403:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575314.051404:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575314.051405:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575314.051408:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575314.051409:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.051410:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575314.051412:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516807424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575339 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575314.051416:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575314.051417:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575314.051418:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575314.051419:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575314.051420:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575314.051421:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575314.051422:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575314.051423:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575314.051424:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575314.051424:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575314.051425:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575314.051427:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575314.051428:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.051429:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.051432:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575314.051433:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.051434:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575314.051435:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575314.560905:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575314.560913:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575314.560915:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575314.560923:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575314.560927:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575314.560932:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.560941:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575314.560943:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575314.560947:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575315.265113:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575315.265117:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575315.265121:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575315.584933:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575315.584941:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575315.584945:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575315.584953:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575315.584961:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575315.584974:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575315.584981:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575315.584990:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575315.585002:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575315.585048:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575315.585069:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575315.585071:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575315.585076:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575315.585080:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575315.585083:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575315.585087:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575315.585089:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575315.585092:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575316.608889:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575316.608899:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575316.608905:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575316.608915:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575316.608923:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575316.608930:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575316.608934:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575316.608940:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575316.608948:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575317.313078:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575317.313082:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575317.313084:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575317.632960:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575317.632966:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575317.632971:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575317.632977:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575317.632983:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575317.633074:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575317.633077:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575317.633078:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575317.633080:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575317.888938:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310677741, at 4310677760 00000100:00000010:0.0:1788575317.888957:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575318.337090:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575318.337095:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575318.337098:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575318.656910:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575318.656918:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575318.656922:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575318.656927:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575318.656930:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575318.656933:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575318.656939:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575318.656942:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575318.656945:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.169367:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575319.169373:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575319.169377:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575319.169386:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516807552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.169398:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575319.169402:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575319.169408:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575319.169413:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575319.169418:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575319.169424:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516807552:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575319.169432:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575319.169439:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575319.169443:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575319.169446:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575319.169452:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575319.169456:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575319.169472:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575319.169478:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575319.169488:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575319.169500:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516807552, portal 25 00000100:00000001:0.0:1788575319.169505:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575319.169513:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788575319.169522:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516807552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.169536:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575319.169543:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575319.169549:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575319.169554:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516807552, offset 0 00000400:00000200:0.0:1788575319.169564:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575319.169578:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575319.169586:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575319.169595:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575319.169603:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575319.169609:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575319.169614:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575319.169625:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547961:547961:256:4294967295] 192.168.202.128@tcp LPNI seq info [547884:547884:8:4294967295] 00000400:00000200:0.0:1788575319.169633:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575319.169649:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575319.169657:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575319.169666:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575319.169678:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575319.169686:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575319.169707:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575319.169712:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575319.169721:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575319.169729:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575319.169736:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575319.169745:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575319.169753:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575319.169758:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516807552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.169776:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575319.169782:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516807552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.169792:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.169794:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575319.169801:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575319.169831:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575319.169835:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310682862 00000100:00000010:0.0:1788575319.169841:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575319.169849:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575319.169859:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310682862 00000100:00000010:0.0:1788575319.169867:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50360. 00000100:00000001:0.0:1788575319.169874:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575319.169883:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575319.169891:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.169899:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575319.169910:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575319/real 1788575319] req@ffff8fc4ac820000 x1875440516807552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575319.169926:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575319.169929:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575319.169934:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575319.169953:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575319.169957:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575319.169962:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575319.169967:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516807552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575319.169984:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516807552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170004:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575319.170007:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170011:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575319.170015:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575319.170024:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516807552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170037:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575319.170043:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516807552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170054:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575319.170058:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170063:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575319.170065:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170070:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575319.170075:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575319.170080:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575319.170086:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575319.170091:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575319.170100:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516807552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575319.170120:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516807552:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575319.170125:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170127:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575319.170134:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516807552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170147:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575319.170150:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575319.170160:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575319.170167:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575319.170171:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575319.170175:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575319.170180:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575319.170183:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788575319.170190:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575319.170194:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575319.170197:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788575319.170201:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575319.170204:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170209:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170218:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575319.170223:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.170228:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575319.170231:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575319.170234:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575319.170239:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516807680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170248:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575319.170250:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575319.170252:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575319.170255:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575319.170258:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575319.170263:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807680:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575319.170270:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575319.170273:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575319.170275:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575319.170278:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575319.170283:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575319.170286:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575319.170295:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575319.170298:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575319.170305:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575319.170311:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516807680, portal 10 00000100:00000001:0.0:1788575319.170320:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575319.170322:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788575319.170331:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516807680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170337:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575319.170339:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575319.170341:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575319.170343:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516807680, offset 0 00000400:00000200:0.0:1788575319.170346:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575319.170349:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575319.170351:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575319.170354:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575319.170356:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575319.170358:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575319.170360:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575319.170362:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547962:547962:256:4294967295] 192.168.202.128@tcp LPNI seq info [547885:547885:8:4294967295] 00000400:00000200:0.0:1788575319.170365:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575319.170370:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575319.170373:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575319.170376:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575319.170378:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575319.170381:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575319.170389:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575319.170390:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575319.170392:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575319.170394:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575319.170397:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575319.170398:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575319.170400:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575319.170402:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516807680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170416:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575319.170418:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516807680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170423:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.170424:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575319.170427:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575319.170430:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575319.170432:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575319.170434:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310679059 00000100:00000001:0.0:1788575319.170436:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310682862 00000100:00000001:0.0:1788575319.170438:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310679059 00000100:00000001:0.0:1788575319.170439:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310682862 00000100:00000001:0.0:1788575319.170441:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575319.170442:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.170443:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575319.170446:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575319/real 1788575319] req@ffff8fc4ac823100 x1875440516807680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575319.170450:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575319.170451:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575319.170452:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575319.170454:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575319.170455:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575319.170456:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575319.170458:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516807680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575319.170463:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516807680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170468:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575319.170473:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170475:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575319.170476:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575319.170479:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516807680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170483:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575319.170486:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516807680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170491:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575319.170492:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170493:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575319.170494:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170495:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575319.170496:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575319.170498:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575319.170500:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575319.170501:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575319.170504:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516807680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575319.170508:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807680:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575319.170511:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575319.170513:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516807808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170518:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575319.170519:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575319.170520:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575319.170521:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575319.170522:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575319.170527:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807808:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575319.170529:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575319.170531:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575319.170532:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575319.170534:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575319.170536:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575319.170537:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575319.170539:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575319.170541:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575319.170543:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575319.170545:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516807808, portal 10 00000100:00000001:0.0:1788575319.170547:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575319.170548:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788575319.170552:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516807808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170557:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575319.170558:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575319.170560:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575319.170561:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516807808, offset 0 00000400:00000200:0.0:1788575319.170564:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575319.170567:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575319.170569:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575319.170571:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575319.170573:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575319.170575:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575319.170577:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575319.170579:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547963:547963:256:4294967295] 192.168.202.128@tcp LPNI seq info [547886:547886:8:4294967295] 00000400:00000200:0.0:1788575319.170582:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575319.170586:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575319.170588:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575319.170590:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575319.170596:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575319.170598:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575319.170605:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575319.170607:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575319.170609:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575319.170611:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575319.170614:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575319.170615:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575319.170616:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575319.170619:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516807808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170624:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575319.170626:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516807808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170631:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.170632:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575319.170634:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575319.170637:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575319.170639:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575319.170640:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310679066 00000100:00000001:0.0:1788575319.170642:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310682862 00000100:00000001:0.0:1788575319.170643:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310679066 00000100:00000001:0.0:1788575319.170645:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310682862 00000100:00000001:0.0:1788575319.170646:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575319.170648:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.170649:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575319.170651:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575319/real 1788575319] req@ffff8fc4ac822a00 x1875440516807808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575319.170655:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575319.170659:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575319.170661:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575319.170662:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575319.170663:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575319.170665:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575319.170667:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516807808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575319.170672:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516807808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170677:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575319.170678:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170679:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575319.170680:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575319.170683:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516807808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170686:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575319.170689:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516807808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170693:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575319.170694:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170695:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575319.170696:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170697:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575319.170699:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575319.170701:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575319.170703:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575319.170704:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575319.170706:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516807808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575319.170718:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807808:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575319.170721:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575319.170723:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516807936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170727:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575319.170728:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575319.170729:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575319.170730:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575319.170732:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575319.170733:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807936:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575319.170735:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575319.170736:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575319.170737:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575319.170738:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575319.170741:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575319.170742:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575319.170745:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575319.170747:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575319.170750:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575319.170751:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516807936, portal 4 00000100:00000001:0.0:1788575319.170753:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575319.170754:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788575319.170758:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516807936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170763:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575319.170764:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575319.170765:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575319.170766:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516807936, offset 0 00000400:00000200:0.0:1788575319.170768:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575319.170774:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575319.170777:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575319.170779:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575319.170780:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575319.170782:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575319.170784:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575319.170786:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547964:547964:256:4294967295] 192.168.202.128@tcp LPNI seq info [547887:547887:8:4294967295] 00000400:00000200:0.0:1788575319.170789:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575319.170793:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575319.170796:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575319.170798:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575319.170800:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575319.170802:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575319.170825:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575319.170827:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575319.170829:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575319.170831:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575319.170834:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575319.170835:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575319.170838:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575319.170842:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516807936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170850:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575319.170854:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516807936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170861:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.170862:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575319.170866:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575319.170869:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575319.170872:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575319.170881:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310679207 00000100:00000001:0.0:1788575319.170884:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310682862 00000100:00000001:0.0:1788575319.170886:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310679207 00000100:00000001:0.0:1788575319.170889:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310682862 00000100:00000001:0.0:1788575319.170891:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575319.170893:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.170896:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575319.170900:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575319/real 1788575319] req@ffff8fc4ac823b80 x1875440516807936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575319.170907:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575319.170909:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575319.170910:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575319.170912:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575319.170913:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575319.170915:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575319.170917:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516807936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575319.170923:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516807936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170928:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575319.170929:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170930:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575319.170931:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575319.170933:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516807936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170949:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575319.170952:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516807936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170957:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575319.170964:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170965:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575319.170966:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170967:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575319.170969:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575319.170971:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575319.170974:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575319.170976:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575319.170978:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516807936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575319.170984:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516807936:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575319.170986:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.170987:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575319.170989:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516807680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.170993:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575319.170994:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575319.170996:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575319.170998:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575319.170999:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575319.171000:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575319.171002:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575319.171003:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788575319.171006:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575319.171007:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575319.171008:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788575319.171010:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575319.171011:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.171012:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575319.171014:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516807808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.171025:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575319.171026:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575319.171027:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575319.171029:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575319.171030:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575319.171030:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575319.171032:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575319.171032:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788575319.171034:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575319.171034:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575319.171035:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788575319.171037:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575319.171037:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.171038:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575319.171040:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516807936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.171044:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575319.171045:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575319.171046:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575319.171047:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575319.171047:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575319.171048:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575319.171049:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575319.171050:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788575319.171051:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575319.171052:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575319.171053:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788575319.171054:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575319.171055:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.171056:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.171058:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575319.171059:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.171060:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575319.171061:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575319.171062:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575319.171067:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516808064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.171072:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575319.171073:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575319.171075:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575319.171076:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575319.171077:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575319.171079:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516808064:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575319.171081:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575319.171083:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575319.171084:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575319.171085:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575319.171087:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575319.171088:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575319.171091:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575319.171092:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575319.171095:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575319.171097:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516808064, portal 4 00000100:00000001:0.0:1788575319.171099:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575319.171099:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788575319.171104:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516808064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.171108:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575319.171110:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575319.171112:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575319.171113:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516808064, offset 0 00000400:00000200:0.0:1788575319.171116:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575319.171118:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575319.171121:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575319.171122:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575319.171124:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575319.171126:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575319.171128:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575319.171133:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547965:547965:256:4294967295] 192.168.202.128@tcp LPNI seq info [547888:547888:8:4294967295] 00000400:00000200:0.0:1788575319.171136:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575319.171141:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575319.171144:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575319.171147:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575319.171149:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575319.171151:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575319.171158:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575319.171160:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575319.171162:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575319.171164:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575319.171166:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575319.171168:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575319.171170:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575319.171172:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516808064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.171177:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575319.171179:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516808064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.171184:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.171185:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575319.171187:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575319.171190:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575319.171192:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575319.171194:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310679209 00000100:00000001:0.0:1788575319.171196:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310682862 00000100:00000001:0.0:1788575319.171197:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310679209 00000100:00000001:0.0:1788575319.171200:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310682862 00000100:00000001:0.0:1788575319.171201:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575319.171206:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.171208:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575319.171212:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575319/real 1788575319] req@ffff8fc4ac821500 x1875440516808064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575319.171218:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575319.171219:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575319.171221:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575319.171222:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575319.171223:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575319.171225:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575319.171227:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516808064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575319.171232:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516808064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.171237:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575319.171238:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.171239:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575319.171240:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575319.171242:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516808064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.171246:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575319.171248:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516808064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.171252:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575319.171253:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.171254:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575319.171255:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.171256:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575319.171258:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575319.171259:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575319.171264:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575319.171266:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575319.171269:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516808064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575319.171274:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516808064:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575319.171276:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.171277:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575319.171279:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516808064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575319.171283:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575319.171283:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575319.171285:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575319.171286:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575319.171288:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575319.171288:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575319.171290:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575319.171291:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788575319.171293:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575319.171294:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575319.171295:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788575319.171298:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575319.171300:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.171301:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.171304:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575319.171305:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.171306:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575319.171308:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.680912:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575319.680939:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575319.680944:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575319.680954:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575319.680963:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575319.680971:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.681002:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575319.681006:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.681009:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575319.681048:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575319.681051:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575319.681053:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575319.681057:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575319.681064:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575319.681066:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.681072:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575319.681075:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575319.681085:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575320.385025:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575320.385029:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575320.385033:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575320.704946:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575320.704953:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575320.704957:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575320.704964:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575320.704972:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575320.704977:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575320.704995:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575320.704999:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575320.705004:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575320.705024:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575320.705029:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575320.705031:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575320.705034:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575320.705037:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575320.705040:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575320.705042:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575320.705044:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575320.705047:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575321.728936:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575321.728945:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575321.728949:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575321.728959:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575321.728967:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575321.728982:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575321.728997:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575321.729001:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575321.729004:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575322.433123:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575322.433126:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575322.433129:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575322.752933:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575322.752943:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575322.752948:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575322.752958:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575322.752966:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575322.752972:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575322.752993:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575322.752996:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575322.753000:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575323.008898:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310682862, at 4310682880 00000100:00000010:0.0:1788575323.008911:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575323.456995:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575323.456998:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575323.457000:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575323.776896:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575323.776905:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575323.776911:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575323.776920:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575323.776926:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575323.776931:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575323.776959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575323.776964:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575323.776969:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.289152:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575324.289157:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575324.289162:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575324.289175:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516808192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.289194:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575324.289206:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575324.289212:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575324.289217:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575324.289224:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575324.289231:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516808192:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575324.289239:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575324.289244:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575324.289248:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575324.289255:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575324.289262:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575324.289267:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575324.289278:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575324.289285:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575324.289294:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575324.289308:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516808192, portal 25 00000100:00000001:0.0:1788575324.289313:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575324.289318:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788575324.289328:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516808192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.289348:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575324.289356:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575324.289361:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575324.289368:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516808192, offset 0 00000400:00000200:0.0:1788575324.289388:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575324.289399:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575324.289409:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575324.289417:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575324.289424:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575324.289430:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575324.289436:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575324.289450:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547966:547966:256:4294967295] 192.168.202.128@tcp LPNI seq info [547889:547889:8:4294967295] 00000400:00000200:0.0:1788575324.289459:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575324.289476:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575324.289495:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575324.289504:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575324.289516:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575324.289522:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575324.289562:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575324.289568:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575324.289576:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575324.289584:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575324.289592:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575324.289597:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575324.289605:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575324.289615:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516808192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.289639:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575324.289647:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516808192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.289668:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.289673:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575324.289681:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575324.289688:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575324.289693:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310687981 00000100:00000010:0.0:1788575324.289700:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575324.289709:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575324.289716:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310687981 00000100:00000010:0.0:1788575324.289720:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50060. 00000100:00000001:0.0:1788575324.289724:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575324.289730:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575324.289734:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.289739:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575324.289748:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575324/real 1788575324] req@ffff8fc4ac821500 x1875440516808192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575324.289800:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575324.289804:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575324.289852:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575324.289859:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575324.289863:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575324.289866:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575324.289871:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516808192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575324.289882:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516808192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.289894:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575324.289898:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.289902:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575324.289906:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575324.289915:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516808192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.289934:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575324.289952:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516808192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.289971:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575324.289978:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.289982:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575324.289986:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.289990:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575324.289994:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575324.289998:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575324.290007:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575324.290012:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575324.290021:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516808192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575324.290038:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516808192:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575324.290044:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.290046:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575324.290049:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516808192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290057:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575324.290059:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575324.290062:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575324.290065:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575324.290067:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575324.290069:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575324.290071:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575324.290074:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788575324.290078:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575324.290080:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575324.290082:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788575324.290085:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575324.290086:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.290088:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.290095:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575324.290096:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.290099:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575324.290100:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575324.290102:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575324.290106:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516808320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290114:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575324.290115:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575324.290118:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575324.290120:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575324.290122:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575324.290125:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516808320:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575324.290131:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575324.290133:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575324.290135:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575324.290137:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575324.290140:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575324.290142:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575324.290146:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575324.290149:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575324.290152:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575324.290155:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516808320, portal 10 00000100:00000001:0.0:1788575324.290158:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575324.290159:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788575324.290165:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516808320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290173:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575324.290175:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575324.290177:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575324.290180:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516808320, offset 0 00000400:00000200:0.0:1788575324.290185:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575324.290189:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575324.290192:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575324.290195:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575324.290198:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575324.290201:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575324.290203:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575324.290207:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547967:547967:256:4294967295] 192.168.202.128@tcp LPNI seq info [547890:547890:8:4294967295] 00000400:00000200:0.0:1788575324.290212:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575324.290219:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575324.290223:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575324.290226:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575324.290230:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575324.290237:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575324.290249:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575324.290251:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575324.290254:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575324.290258:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575324.290261:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575324.290263:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575324.290266:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575324.290270:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516808320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290278:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575324.290281:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516808320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290288:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.290290:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575324.290293:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575324.290296:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575324.290300:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575324.290302:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310684179 00000100:00000001:0.0:1788575324.290305:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310687981 00000100:00000001:0.0:1788575324.290307:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310684179 00000100:00000001:0.0:1788575324.290309:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310687981 00000100:00000001:0.0:1788575324.290311:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575324.290313:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.290315:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575324.290319:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575324/real 1788575324] req@ffff8fc4ac823b80 x1875440516808320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575324.290327:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575324.290328:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575324.290335:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575324.290337:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575324.290339:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575324.290342:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575324.290346:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516808320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575324.290355:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516808320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290362:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575324.290364:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.290366:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575324.290367:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575324.290371:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516808320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290378:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575324.290382:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516808320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290390:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575324.290393:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.290395:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575324.290396:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.290398:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575324.290400:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575324.290402:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575324.290405:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575324.290408:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575324.290412:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516808320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575324.290420:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516808320:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575324.290427:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575324.290431:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516808448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290438:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575324.290440:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575324.290442:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575324.290444:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575324.290446:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575324.290449:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516808448:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575324.290452:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575324.290454:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575324.290456:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575324.290458:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575324.290461:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575324.290463:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575324.290466:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575324.290468:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575324.290472:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575324.290475:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516808448, portal 10 00000100:00000001:0.0:1788575324.290477:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575324.290478:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788575324.290484:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516808448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290491:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575324.290493:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575324.290496:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575324.290498:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516808448, offset 0 00000400:00000200:0.0:1788575324.290502:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575324.290506:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575324.290509:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575324.290512:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575324.290518:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575324.290521:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575324.290523:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575324.290527:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547968:547968:256:4294967295] 192.168.202.128@tcp LPNI seq info [547891:547891:8:4294967295] 00000400:00000200:0.0:1788575324.290531:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575324.290538:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575324.290543:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575324.290546:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575324.290550:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575324.290553:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575324.290564:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575324.290566:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575324.290569:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575324.290573:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575324.290575:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575324.290577:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575324.290580:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575324.290584:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516808448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290593:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575324.290596:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516808448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290603:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.290605:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575324.290608:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575324.290611:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575324.290614:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575324.290616:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310684186 00000100:00000001:0.0:1788575324.290619:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310687981 00000100:00000001:0.0:1788575324.290621:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310684186 00000100:00000001:0.0:1788575324.290627:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310687981 00000100:00000001:0.0:1788575324.290629:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575324.290632:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.290634:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575324.290638:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575324/real 1788575324] req@ffff8fc4ac822a00 x1875440516808448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575324.290646:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575324.290648:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575324.290650:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575324.290652:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575324.290654:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575324.290656:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575324.290660:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516808448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575324.290669:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516808448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290676:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575324.290677:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.290679:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575324.290681:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575324.290684:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516808448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290692:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575324.290696:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516808448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290703:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575324.290705:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.290707:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575324.290708:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.290713:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575324.290716:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575324.290719:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575324.290722:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575324.290725:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575324.290729:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516808448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575324.290738:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516808448:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575324.290742:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575324.290745:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516808576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290752:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575324.290754:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575324.290756:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575324.290758:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575324.290760:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575324.290763:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516808576:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575324.290766:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575324.290767:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575324.290769:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575324.290771:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575324.290786:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575324.290789:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575324.290793:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575324.290796:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575324.290799:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575324.290802:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516808576, portal 4 00000100:00000001:0.0:1788575324.290805:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575324.290806:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788575324.290830:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516808576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290842:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575324.290845:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575324.290847:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575324.290850:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516808576, offset 0 00000400:00000200:0.0:1788575324.290854:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575324.290858:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575324.290862:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575324.290864:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575324.290867:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575324.290870:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575324.290872:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575324.290876:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547969:547969:256:4294967295] 192.168.202.128@tcp LPNI seq info [547892:547892:8:4294967295] 00000400:00000200:0.0:1788575324.290880:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575324.290887:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575324.290891:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575324.290894:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575324.290898:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575324.290901:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575324.290912:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575324.290913:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575324.290916:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575324.290920:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575324.290923:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575324.290925:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575324.290928:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575324.290931:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516808576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290940:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575324.290943:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516808576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.290962:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.290964:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575324.290969:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575324.290972:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575324.290976:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575324.290978:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310684327 00000100:00000001:0.0:1788575324.290980:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310687981 00000100:00000001:0.0:1788575324.290983:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310684327 00000100:00000001:0.0:1788575324.290985:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310687981 00000100:00000001:0.0:1788575324.290987:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575324.290990:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.290992:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575324.290996:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575324/real 1788575324] req@ffff8fc4ac823100 x1875440516808576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575324.291004:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575324.291006:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575324.291008:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575324.291010:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575324.291012:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575324.291014:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575324.291017:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516808576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575324.291027:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516808576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291034:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575324.291035:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.291037:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575324.291038:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575324.291042:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516808576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291055:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575324.291060:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516808576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291068:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575324.291070:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.291072:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575324.291074:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.291075:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575324.291077:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575324.291080:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575324.291083:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575324.291085:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575324.291089:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516808576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575324.291098:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516808576:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575324.291102:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575324.291105:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516808704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291113:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575324.291114:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575324.291116:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575324.291118:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575324.291120:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575324.291123:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516808704:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575324.291126:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575324.291128:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575324.291133:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575324.291135:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575324.291137:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575324.291139:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575324.291143:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575324.291145:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575324.291149:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575324.291151:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516808704, portal 4 00000100:00000001:0.0:1788575324.291153:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575324.291155:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788575324.291161:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516808704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291168:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575324.291170:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575324.291172:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575324.291175:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516808704, offset 0 00000400:00000200:0.0:1788575324.291179:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575324.291182:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575324.291186:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575324.291189:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575324.291192:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575324.291194:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575324.291197:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575324.291200:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547970:547970:256:4294967295] 192.168.202.128@tcp LPNI seq info [547893:547893:8:4294967295] 00000400:00000200:0.0:1788575324.291205:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575324.291212:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575324.291217:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575324.291220:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575324.291224:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575324.291227:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575324.291238:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575324.291243:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575324.291246:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575324.291250:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575324.291253:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575324.291255:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575324.291258:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575324.291261:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516808704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291270:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575324.291273:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516808704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291280:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.291282:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575324.291285:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575324.291288:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575324.291291:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575324.291293:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310684329 00000100:00000001:0.0:1788575324.291296:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310687981 00000100:00000001:0.0:1788575324.291298:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310684329 00000100:00000001:0.0:1788575324.291300:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310687981 00000100:00000001:0.0:1788575324.291302:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575324.291304:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.291306:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575324.291310:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575324/real 1788575324] req@ffff8fc4ac820000 x1875440516808704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575324.291317:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575324.291319:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575324.291321:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575324.291323:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575324.291325:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575324.291330:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575324.291334:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516808704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575324.291343:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516808704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291350:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575324.291351:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.291353:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575324.291355:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575324.291358:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516808704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291366:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575324.291370:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516808704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291377:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575324.291380:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.291381:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575324.291383:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.291385:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575324.291387:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575324.291389:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575324.291392:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575324.291394:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575324.291398:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516808704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575324.291407:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516808704:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575324.291410:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.291412:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575324.291419:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516808320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291426:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575324.291428:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575324.291430:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575324.291432:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575324.291433:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575324.291435:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575324.291437:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575324.291439:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788575324.291442:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575324.291443:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575324.291445:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788575324.291448:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575324.291449:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.291451:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575324.291455:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516808448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291462:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575324.291463:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575324.291465:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575324.291466:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575324.291468:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575324.291469:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575324.291471:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575324.291472:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788575324.291474:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575324.291475:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575324.291477:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788575324.291479:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575324.291480:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.291482:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575324.291485:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516808576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291491:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575324.291496:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575324.291497:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575324.291499:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575324.291501:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575324.291502:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575324.291504:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575324.291506:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788575324.291508:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575324.291509:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575324.291511:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788575324.291513:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575324.291514:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.291516:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575324.291519:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516808704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575324.291526:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575324.291527:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575324.291528:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575324.291530:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575324.291531:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575324.291533:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575324.291535:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575324.291536:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788575324.291538:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575324.291539:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575324.291541:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788575324.291543:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575324.291544:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.291546:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.291549:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575324.291551:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.291553:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575324.291554:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575324.800922:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575324.800929:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575324.800938:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575324.800945:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575324.800964:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575324.800971:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.801004:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575324.801009:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575324.801024:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575325.505023:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575325.505026:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575325.505029:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575325.824963:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575325.824978:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575325.824984:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575325.824991:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575325.825002:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575325.825011:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575325.825020:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575325.825024:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575325.825028:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575326.848937:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575326.848946:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575326.848949:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575326.848955:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575326.848962:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575326.848967:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575326.848972:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575326.848977:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575326.848982:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575326.849024:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575326.849031:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575326.849033:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575326.849035:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575326.849038:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575326.849040:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575326.849042:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575326.849044:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575326.849047:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575327.553083:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575327.553088:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575327.553093:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575327.872906:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575327.872913:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575327.872918:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575327.872924:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575327.872930:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575327.872939:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575327.872963:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575327.872967:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575327.872970:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575327.873002:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575327.873007:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575327.873009:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575327.873012:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575327.873014:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575327.873017:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575327.873019:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575327.873021:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575327.873025:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575328.128914:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310687981, at 4310688000 00000100:00000010:0.0:1788575328.128933:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575328.577046:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575328.577050:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575328.577053:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575328.896944:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575328.896954:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575328.896958:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575328.896966:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575328.896974:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575328.896980:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575328.896994:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575328.896996:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575328.896998:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.409257:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575329.409264:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575329.409268:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575329.409279:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516808832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.409318:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575329.409321:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575329.409328:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575329.409333:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575329.409340:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575329.409347:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516808832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575329.409355:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575329.409360:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575329.409364:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575329.409369:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575329.409378:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575329.409383:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575329.409397:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575329.409405:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575329.409417:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575329.409428:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516808832, portal 25 00000100:00000001:0.0:1788575329.409434:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575329.409439:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788575329.409453:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516808832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.409470:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575329.409485:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575329.409493:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575329.409502:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516808832, offset 0 00000400:00000200:0.0:1788575329.409518:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575329.409535:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575329.409548:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575329.409560:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575329.409577:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575329.409586:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575329.409593:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575329.409629:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547971:547971:256:4294967295] 192.168.202.128@tcp LPNI seq info [547894:547894:8:4294967295] 00000400:00000200:0.0:1788575329.409640:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575329.409660:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575329.409670:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575329.409684:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575329.409700:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575329.409708:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788575329.409750:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575329.409758:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575329.409771:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575329.409784:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575329.409795:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575329.409801:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575329.409856:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575329.409867:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516808832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.409893:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575329.409903:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516808832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.409923:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.409930:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575329.409941:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575329.409953:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575329.409959:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310693102 00000100:00000010:0.0:1788575329.409969:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575329.409980:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575329.409993:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310693102 00000100:00000010:0.0:1788575329.410002:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50600. 00000100:00000001:0.0:1788575329.410010:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575329.410017:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575329.410032:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.410039:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575329.410048:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575329/real 1788575329] req@ffff8fc4ac822d80 x1875440516808832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575329.410074:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575329.410078:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575329.410084:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575329.410090:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575329.410094:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575329.410100:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575329.410108:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516808832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575329.410142:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516808832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410162:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575329.410167:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.410173:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575329.410177:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575329.410186:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516808832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410205:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575329.410217:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516808832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410234:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575329.410240:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.410244:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575329.410248:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.410255:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575329.410261:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575329.410269:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575329.410279:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575329.410293:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575329.410303:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516808832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575329.410320:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516808832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575329.410332:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.410338:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575329.410345:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516808832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410362:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575329.410366:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575329.410370:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575329.410377:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575329.410380:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575329.410382:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575329.410385:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575329.410389:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788575329.410395:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575329.410398:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575329.410400:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788575329.410405:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575329.410407:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.410410:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.410419:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575329.410423:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.410425:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575329.410427:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575329.410429:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575329.410434:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516808960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410443:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575329.410444:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575329.410448:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575329.410451:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575329.410459:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575329.410464:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516808960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575329.410468:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575329.410471:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575329.410474:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575329.410477:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575329.410481:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575329.410483:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575329.410489:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575329.410493:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575329.410500:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575329.410505:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516808960, portal 10 00000100:00000001:0.0:1788575329.410507:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575329.410510:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788575329.410519:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516808960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410526:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575329.410528:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575329.410531:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575329.410533:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516808960, offset 0 00000400:00000200:0.0:1788575329.410537:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575329.410543:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575329.410548:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575329.410551:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575329.410555:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575329.410558:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575329.410561:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575329.410565:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547972:547972:256:4294967295] 192.168.202.128@tcp LPNI seq info [547895:547895:8:4294967295] 00000400:00000200:0.0:1788575329.410570:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575329.410578:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575329.410584:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575329.410596:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575329.410602:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575329.410606:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788575329.410632:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575329.410634:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575329.410638:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575329.410643:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575329.410648:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575329.410650:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575329.410653:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575329.410657:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516808960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410666:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575329.410670:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516808960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410676:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.410678:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575329.410682:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575329.410687:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575329.410691:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575329.410693:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310689299 00000100:00000001:0.0:1788575329.410697:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310693102 00000100:00000001:0.0:1788575329.410699:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310689299 00000100:00000001:0.0:1788575329.410702:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310693102 00000100:00000001:0.0:1788575329.410704:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575329.410707:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.410709:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575329.410714:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575329/real 1788575329] req@ffff8fc4ac821f80 x1875440516808960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575329.410735:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575329.410737:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575329.410739:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575329.410743:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575329.410745:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575329.410748:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575329.410752:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516808960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575329.410760:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516808960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410767:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575329.410769:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.410771:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575329.410772:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575329.410776:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516808960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410783:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575329.410788:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516808960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410795:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575329.410798:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.410800:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575329.410802:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.410803:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575329.410805:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575329.410834:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575329.410839:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575329.410842:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575329.410853:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516808960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575329.410862:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516808960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575329.410867:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575329.410872:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516809088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410879:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575329.410881:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575329.410884:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575329.410888:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575329.410891:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575329.410894:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575329.410898:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575329.410901:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575329.410904:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575329.410906:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575329.410911:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575329.410913:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575329.410918:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575329.410921:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575329.410925:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575329.410929:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516809088, portal 10 00000100:00000001:0.0:1788575329.410932:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575329.410933:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788575329.410941:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516809088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.410949:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575329.410951:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575329.410954:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575329.410957:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516809088, offset 0 00000400:00000200:0.0:1788575329.410968:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575329.410974:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575329.410977:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575329.410980:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575329.410984:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575329.410986:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575329.410988:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575329.410991:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547973:547973:256:4294967295] 192.168.202.128@tcp LPNI seq info [547896:547896:8:4294967295] 00000400:00000200:0.0:1788575329.410995:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575329.411001:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575329.411006:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575329.411009:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575329.411013:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575329.411015:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788575329.411026:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575329.411028:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575329.411032:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575329.411035:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575329.411039:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575329.411041:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575329.411043:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575329.411047:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516809088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411056:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575329.411059:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516809088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411066:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.411067:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575329.411071:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575329.411074:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575329.411078:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575329.411087:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310689307 00000100:00000001:0.0:1788575329.411090:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310693102 00000100:00000001:0.0:1788575329.411093:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310689307 00000100:00000001:0.0:1788575329.411095:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310693102 00000100:00000001:0.0:1788575329.411098:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575329.411100:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.411103:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575329.411108:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575329/real 1788575329] req@ffff8fc4ac823480 x1875440516809088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575329.411117:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575329.411119:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575329.411121:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575329.411123:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575329.411125:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575329.411128:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575329.411132:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516809088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575329.411140:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516809088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411148:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575329.411149:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411152:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575329.411153:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575329.411158:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516809088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411166:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575329.411170:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516809088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411178:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575329.411186:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411188:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575329.411189:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411191:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575329.411195:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575329.411198:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575329.411202:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575329.411204:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575329.411209:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516809088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575329.411218:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575329.411223:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575329.411227:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516809216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411235:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575329.411236:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575329.411239:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575329.411241:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575329.411243:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575329.411246:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575329.411250:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575329.411252:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575329.411254:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575329.411256:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575329.411260:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575329.411262:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575329.411266:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575329.411269:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575329.411273:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575329.411277:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516809216, portal 4 00000100:00000001:0.0:1788575329.411284:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575329.411285:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788575329.411292:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516809216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411298:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575329.411301:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575329.411303:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575329.411306:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516809216, offset 0 00000400:00000200:0.0:1788575329.411310:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575329.411314:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575329.411319:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575329.411322:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575329.411325:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575329.411328:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575329.411330:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575329.411333:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547974:547974:256:4294967295] 192.168.202.128@tcp LPNI seq info [547897:547897:8:4294967295] 00000400:00000200:0.0:1788575329.411339:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575329.411351:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575329.411354:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575329.411358:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575329.411360:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575329.411363:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788575329.411370:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575329.411372:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575329.411374:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575329.411377:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575329.411379:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575329.411381:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575329.411383:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575329.411385:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516809216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411403:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575329.411406:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516809216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411411:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.411412:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575329.411415:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575329.411418:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575329.411420:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575329.411422:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310689447 00000100:00000001:0.0:1788575329.411424:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310693102 00000100:00000001:0.0:1788575329.411426:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310689447 00000100:00000001:0.0:1788575329.411427:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310693102 00000100:00000001:0.0:1788575329.411429:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575329.411430:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.411431:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575329.411434:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575329/real 1788575329] req@ffff8fc4ac820700 x1875440516809216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575329.411438:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575329.411440:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575329.411441:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575329.411443:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575329.411444:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575329.411446:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575329.411448:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516809216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575329.411453:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516809216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411457:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575329.411461:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411462:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575329.411464:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575329.411466:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516809216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411471:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575329.411474:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516809216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411478:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575329.411480:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411481:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575329.411482:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411483:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575329.411484:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575329.411486:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575329.411488:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575329.411489:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575329.411491:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516809216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575329.411496:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575329.411499:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575329.411501:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516809344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411508:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575329.411509:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575329.411512:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575329.411514:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575329.411516:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575329.411525:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575329.411555:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575329.411558:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575329.411560:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575329.411562:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575329.411566:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575329.411568:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575329.411572:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575329.411574:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575329.411577:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575329.411580:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516809344, portal 4 00000100:00000001:0.0:1788575329.411587:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575329.411589:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788575329.411597:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516809344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411604:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575329.411607:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575329.411609:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575329.411623:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516809344, offset 0 00000400:00000200:0.0:1788575329.411628:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575329.411632:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575329.411636:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575329.411639:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575329.411642:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575329.411645:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575329.411648:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575329.411651:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547975:547975:256:4294967295] 192.168.202.128@tcp LPNI seq info [547898:547898:8:4294967295] 00000400:00000200:0.0:1788575329.411655:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575329.411662:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575329.411667:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575329.411670:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575329.411677:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575329.411680:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788575329.411692:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575329.411695:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575329.411698:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575329.411704:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575329.411708:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575329.411710:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575329.411715:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575329.411720:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516809344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411729:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575329.411732:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516809344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411739:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.411741:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575329.411745:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575329.411748:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575329.411752:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575329.411754:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310689449 00000100:00000001:0.0:1788575329.411757:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310693102 00000100:00000001:0.0:1788575329.411759:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310689449 00000100:00000001:0.0:1788575329.411761:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310693102 00000100:00000001:0.0:1788575329.411764:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575329.411766:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.411768:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575329.411771:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575329/real 1788575329] req@ffff8fc4ac822300 x1875440516809344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575329.411777:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575329.411785:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575329.411787:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575329.411789:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575329.411791:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575329.411794:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575329.411798:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516809344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575329.411834:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516809344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411843:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575329.411845:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411846:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575329.411848:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575329.411851:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516809344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411856:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575329.411859:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516809344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411864:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575329.411865:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411866:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575329.411867:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411868:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575329.411870:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575329.411872:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575329.411874:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575329.411875:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575329.411878:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516809344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575329.411892:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575329.411895:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411897:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575329.411899:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516808960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411904:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575329.411906:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575329.411907:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575329.411910:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575329.411911:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575329.411913:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575329.411915:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575329.411917:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788575329.411919:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575329.411921:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575329.411922:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788575329.411925:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575329.411926:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411927:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575329.411929:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516809088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411934:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575329.411935:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575329.411936:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575329.411939:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575329.411939:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575329.411940:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575329.411941:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575329.411942:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788575329.411943:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575329.411944:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575329.411945:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788575329.411949:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575329.411950:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411951:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575329.411954:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516809216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411958:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575329.411959:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575329.411960:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575329.411961:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575329.411962:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575329.411962:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575329.411963:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575329.411964:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788575329.411966:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575329.411966:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575329.411967:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788575329.411968:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575329.411969:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411970:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575329.411972:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516809344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575329.411975:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575329.411976:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575329.411977:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575329.411978:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575329.411979:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575329.411979:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575329.411980:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575329.411981:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788575329.411982:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575329.411983:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575329.411984:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788575329.411985:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575329.411986:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411987:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.411992:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575329.411997:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.411998:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575329.412000:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.920930:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575329.920942:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575329.920949:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575329.920960:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575329.920968:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575329.920973:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.920980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575329.920983:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.920988:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575329.921022:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575329.921026:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575329.921027:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575329.921029:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575329.921032:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575329.921035:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.921037:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575329.921039:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575329.921041:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575330.625070:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575330.625075:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575330.625080:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788575332.672985:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575332.672990:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575332.672993:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575332.992958:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575332.992972:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575332.992983:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575332.993003:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575332.993017:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575332.993021:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575332.993028:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575332.993032:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575332.993036:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575333.248911:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310693102, at 4310693120 00000100:00000010:0.0:1788575333.248928:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575333.697063:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575333.697068:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575333.697072:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575334.016944:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575334.016952:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575334.016959:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575334.016973:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575334.016986:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575334.016993:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.016997:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575334.017000:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.017004:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.017081:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575334.017084:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575334.017092:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575334.017095:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575334.017097:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575334.017099:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.017101:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575334.017103:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.017106:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.529235:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575334.529243:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575334.529248:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575334.529262:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516809472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.529274:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575334.529282:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575334.529291:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575334.529298:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575334.529306:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575334.529318:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516809472:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575334.529330:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575334.529337:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575334.529345:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575334.529352:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575334.529375:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575334.529381:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575334.529395:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575334.529403:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575334.529424:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575334.529434:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516809472, portal 25 00000100:00000001:0.0:1788575334.529439:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575334.529446:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788575334.529463:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516809472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.529474:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575334.529477:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575334.529479:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575334.529482:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516809472, offset 0 00000400:00000200:0.0:1788575334.529494:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575334.529510:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575334.529531:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575334.529542:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575334.529552:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575334.529562:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575334.529572:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575334.529580:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547976:547976:256:4294967295] 192.168.202.128@tcp LPNI seq info [547899:547899:8:4294967295] 00000400:00000200:0.0:1788575334.529588:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575334.529613:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575334.529630:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575334.529644:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575334.529655:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575334.529663:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575334.529676:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575334.529679:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575334.529686:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575334.529702:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575334.529713:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575334.529716:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575334.529726:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575334.529740:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516809472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.529772:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575334.529781:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516809472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.529830:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.529835:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575334.529843:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575334.529863:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575334.529876:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310698222 00000100:00000010:0.0:1788575334.529895:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575334.529907:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575334.529919:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310698222 00000100:00000010:0.0:1788575334.529927:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50840. 00000100:00000001:0.0:1788575334.529933:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575334.529937:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575334.529947:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.529959:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575334.529972:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575334/real 1788575334] req@ffff8fc4ac821c00 x1875440516809472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575334.529991:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575334.529994:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575334.529999:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575334.530004:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575334.530010:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575334.530021:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575334.530032:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516809472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575334.530095:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516809472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.530118:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575334.530123:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.530130:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575334.530136:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575334.530152:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516809472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.530191:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575334.530203:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516809472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.530234:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575334.530241:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.530244:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575334.530247:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.530252:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575334.530254:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575334.530261:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575334.530274:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575334.530284:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575334.530300:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516809472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575334.530331:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516809472:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575334.530348:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.530356:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575334.530369:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516809472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.530410:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575334.530416:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575334.530423:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575334.530434:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575334.530440:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575334.530450:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575334.530461:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575334.530466:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788575334.530477:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575334.530481:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575334.530486:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788575334.530496:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575334.530499:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.530505:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.530515:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575334.530535:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.530543:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575334.530546:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575334.530547:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575334.530552:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516809600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.530566:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575334.530570:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575334.530573:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575334.530578:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575334.530586:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575334.530599:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809600:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575334.530606:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575334.530611:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575334.530614:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575334.530618:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575334.530623:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575334.530627:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575334.530636:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575334.530649:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575334.530658:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575334.530663:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516809600, portal 10 00000100:00000001:0.0:1788575334.530669:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575334.530672:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788575334.530686:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516809600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.530704:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575334.530713:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575334.530718:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575334.530724:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516809600, offset 0 00000400:00000200:0.0:1788575334.530742:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575334.530752:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575334.530761:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575334.530765:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575334.530772:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575334.530776:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575334.530781:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575334.530786:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547977:547977:256:4294967295] 192.168.202.128@tcp LPNI seq info [547900:547900:8:4294967295] 00000400:00000200:0.0:1788575334.530790:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575334.530801:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575334.530848:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575334.530861:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575334.530876:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575334.530885:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575334.530923:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575334.530925:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575334.530930:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575334.530936:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575334.530939:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575334.530941:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575334.530951:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575334.530956:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516809600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.530962:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575334.530964:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516809600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.530969:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.530970:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575334.530973:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575334.530976:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575334.530978:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575334.530980:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310694420 00000100:00000001:0.0:1788575334.530982:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310698222 00000100:00000001:0.0:1788575334.530984:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310694420 00000100:00000001:0.0:1788575334.530985:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310698222 00000100:00000001:0.0:1788575334.530987:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575334.530988:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.530989:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575334.530992:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575334/real 1788575334] req@ffff8fc4ac820a80 x1875440516809600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575334.530997:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575334.530998:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575334.530999:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575334.531001:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575334.531002:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575334.531003:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575334.531005:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516809600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575334.531011:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516809600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531022:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575334.531023:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531024:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575334.531025:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575334.531028:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516809600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531032:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575334.531035:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516809600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531040:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575334.531042:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531043:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575334.531043:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531045:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575334.531046:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575334.531048:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575334.531050:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575334.531052:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575334.531054:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516809600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575334.531059:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809600:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575334.531062:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575334.531064:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516809728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531069:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575334.531070:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575334.531075:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575334.531077:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575334.531078:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575334.531080:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809728:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575334.531083:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575334.531085:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575334.531086:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575334.531087:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575334.531090:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575334.531091:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575334.531093:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575334.531095:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575334.531097:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575334.531099:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516809728, portal 10 00000100:00000001:0.0:1788575334.531101:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575334.531102:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788575334.531106:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516809728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531111:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575334.531112:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575334.531113:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575334.531115:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516809728, offset 0 00000400:00000200:0.0:1788575334.531118:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575334.531120:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575334.531123:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575334.531126:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575334.531129:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575334.531131:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575334.531134:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575334.531137:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547978:547978:256:4294967295] 192.168.202.128@tcp LPNI seq info [547901:547901:8:4294967295] 00000400:00000200:0.0:1788575334.531142:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575334.531149:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575334.531160:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575334.531163:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575334.531167:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575334.531170:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575334.531179:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575334.531182:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575334.531185:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575334.531189:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575334.531192:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575334.531194:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575334.531197:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575334.531201:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516809728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531210:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575334.531213:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516809728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531221:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.531223:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575334.531227:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575334.531230:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575334.531232:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575334.531234:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310694427 00000100:00000001:0.0:1788575334.531236:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310698222 00000100:00000001:0.0:1788575334.531237:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310694427 00000100:00000001:0.0:1788575334.531239:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310698222 00000100:00000001:0.0:1788575334.531240:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575334.531241:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.531243:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575334.531245:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575334/real 1788575334] req@ffff8fc4ac823800 x1875440516809728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575334.531258:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575334.531258:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575334.531260:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575334.531261:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575334.531263:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575334.531264:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575334.531267:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516809728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575334.531272:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516809728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531276:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575334.531277:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531278:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575334.531279:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575334.531281:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516809728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531285:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575334.531288:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516809728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531292:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575334.531293:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531294:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575334.531295:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531296:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575334.531298:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575334.531300:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575334.531302:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575334.531305:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575334.531314:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516809728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575334.531322:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809728:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575334.531326:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575334.531331:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516809856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531339:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575334.531341:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575334.531343:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575334.531345:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575334.531347:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575334.531351:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809856:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575334.531354:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575334.531356:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575334.531357:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575334.531359:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575334.531361:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575334.531363:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575334.531365:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575334.531367:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575334.531370:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575334.531372:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516809856, portal 4 00000100:00000001:0.0:1788575334.531373:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575334.531374:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788575334.531379:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516809856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531383:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575334.531384:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575334.531386:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575334.531387:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516809856, offset 0 00000400:00000200:0.0:1788575334.531395:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575334.531397:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575334.531400:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575334.531402:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575334.531404:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575334.531406:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575334.531407:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575334.531409:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547979:547979:256:4294967295] 192.168.202.128@tcp LPNI seq info [547902:547902:8:4294967295] 00000400:00000200:0.0:1788575334.531412:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575334.531417:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575334.531420:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575334.531422:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575334.531425:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575334.531427:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575334.531433:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575334.531435:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575334.531437:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575334.531439:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575334.531441:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575334.531442:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575334.531444:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575334.531446:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516809856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531452:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575334.531454:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516809856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531458:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.531459:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575334.531461:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575334.531463:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575334.531465:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575334.531470:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310694567 00000100:00000001:0.0:1788575334.531471:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310698222 00000100:00000001:0.0:1788575334.531473:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310694567 00000100:00000001:0.0:1788575334.531474:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310698222 00000100:00000001:0.0:1788575334.531476:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575334.531477:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.531478:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575334.531481:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575334/real 1788575334] req@ffff8fc4ac821880 x1875440516809856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575334.531486:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575334.531487:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575334.531489:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575334.531490:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575334.531491:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575334.531493:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575334.531495:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516809856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575334.531500:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516809856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531504:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575334.531505:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531506:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575334.531507:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575334.531509:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516809856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531513:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575334.531515:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516809856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531532:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575334.531539:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531540:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575334.531541:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531542:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575334.531543:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575334.531545:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575334.531547:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575334.531549:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575334.531552:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516809856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575334.531557:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809856:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575334.531559:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531561:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575334.531562:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516809600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531567:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575334.531568:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575334.531569:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575334.531571:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575334.531573:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575334.531574:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575334.531575:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575334.531577:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788575334.531578:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575334.531580:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575334.531581:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788575334.531583:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575334.531584:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531585:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575334.531587:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516809728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531598:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575334.531599:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575334.531600:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575334.531602:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575334.531602:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575334.531603:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575334.531604:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575334.531605:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788575334.531607:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575334.531608:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575334.531609:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788575334.531610:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575334.531611:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531612:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575334.531614:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516809856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531618:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575334.531618:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575334.531619:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575334.531620:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575334.531621:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575334.531622:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575334.531623:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575334.531623:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862ac00. 02000000:00000001:0.0:1788575334.531625:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575334.531625:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575334.531626:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788575334.531627:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575334.531628:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531629:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531632:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575334.531633:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.531634:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575334.531635:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575334.531636:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575334.531642:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516809984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531647:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575334.531648:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575334.531650:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575334.531651:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575334.531652:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575334.531654:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809984:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575334.531656:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575334.531658:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575334.531659:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575334.531660:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575334.531662:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575334.531663:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575334.531666:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575334.531668:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575334.531670:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575334.531672:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516809984, portal 4 00000100:00000001:0.0:1788575334.531674:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575334.531675:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788575334.531679:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516809984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531683:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575334.531684:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575334.531686:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575334.531688:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516809984, offset 0 00000400:00000200:0.0:1788575334.531690:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575334.531692:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575334.531695:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575334.531697:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575334.531700:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575334.531702:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575334.531704:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575334.531714:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [547980:547980:256:4294967295] 192.168.202.128@tcp LPNI seq info [547903:547903:8:4294967295] 00000400:00000200:0.0:1788575334.531719:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575334.531725:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575334.531731:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575334.531734:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575334.531738:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575334.531741:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575334.531752:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575334.531754:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575334.531756:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575334.531758:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575334.531761:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575334.531762:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575334.531764:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575334.531767:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516809984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531772:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575334.531774:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516809984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531779:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.531780:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575334.531782:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575334.531784:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575334.531786:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575334.531788:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310694569 00000100:00000001:0.0:1788575334.531789:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310698222 00000100:00000001:0.0:1788575334.531791:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310694569 00000100:00000001:0.0:1788575334.531793:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310698222 00000100:00000001:0.0:1788575334.531794:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575334.531800:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.531802:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575334.531804:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575334/real 1788575334] req@ffff8fc4ac820e00 x1875440516809984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575334.531828:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575334.531830:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575334.531831:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575334.531834:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575334.531836:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575334.531838:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575334.531842:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516809984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575334.531851:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516809984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531858:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575334.531860:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531862:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575334.531863:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575334.531868:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516809984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531875:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575334.531880:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516809984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531885:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575334.531887:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531888:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575334.531889:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531890:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575334.531892:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575334.531893:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575334.531900:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575334.531902:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575334.531904:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516809984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575334.531910:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516809984:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575334.531912:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531913:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575334.531915:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516809984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575334.531919:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575334.531920:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575334.531921:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575334.531923:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575334.531924:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575334.531925:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575334.531926:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575334.531927:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788575334.531929:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575334.531930:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575334.531931:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788575334.531932:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575334.531933:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531934:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575334.531937:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575334.531938:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575334.531939:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575334.531941:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575335.040924:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575335.040932:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575335.040936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575335.040944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575335.040949:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575335.040956:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575335.041006:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575335.041016:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575335.041028:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575335.745061:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575335.745068:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575335.745074:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575336.064980:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575336.064987:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575336.064991:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575336.064999:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575336.065010:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575336.065018:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575336.065026:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575336.065035:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575336.065042:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575337.088976:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575337.088991:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575337.088998:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575337.089010:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575337.089021:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575337.089031:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575337.089053:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575337.089061:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575337.089068:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575337.793015:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575337.793019:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575337.793022:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575338.112945:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575338.112955:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575338.112959:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575338.112963:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575338.112969:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575338.112974:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575338.112977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575338.112979:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575338.112982:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575338.368925:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310698222, at 4310698240 00000100:00000010:0.0:1788575338.368945:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575338.817027:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575338.817031:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575338.817034:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575339.136931:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575339.136938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575339.136941:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575339.136949:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575339.136957:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575339.136963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575339.136973:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575339.136978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575339.136984:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575339.649137:0:2391:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1788575340.160913:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575340.160919:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575340.160925:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575340.160932:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575340.160938:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575340.160945:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575340.160966:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575340.160970:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575340.160974:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575340.864983:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575340.864988:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575340.864991:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575341.185361:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575341.185366:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575341.185368:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575341.185372:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575341.185376:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575341.185381:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575341.185386:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575341.185391:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575341.185396:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575341.185417:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575341.185435:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575341.185436:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575341.185439:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575341.185449:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575341.185452:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575341.185455:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575341.185457:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575341.185459:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575342.208932:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575342.208939:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575342.208942:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575342.208947:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575342.208954:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575342.208959:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575342.208965:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575342.208970:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575342.208977:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575342.913071:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575342.913075:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575342.913078:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575343.232917:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575343.232926:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575343.232930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575343.232938:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575343.232947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575343.232951:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575343.232957:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575343.232961:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575343.232965:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575343.488910:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310703341, at 4310703360 00000100:00000010:0.0:1788575343.488936:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf060. 00000020:00000001:0.0:1788575343.936976:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575343.936979:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575343.936982:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575344.256986:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575344.256992:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575344.256994:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575344.257000:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575344.257008:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575344.257013:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575344.257031:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575344.257034:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575344.257036:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575345.280911:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575345.280916:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575345.280918:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575345.280922:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575345.280924:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575345.280927:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575345.280939:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575345.280943:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575345.280950:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575345.985005:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575345.985008:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575345.985011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575346.304963:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575346.304980:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575346.304989:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575346.305005:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575346.305016:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575346.305032:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575346.305052:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575346.305057:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575346.305062:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575347.328941:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575347.328951:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575347.328954:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575347.328959:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575347.328969:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575347.328977:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575347.328981:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575347.328995:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575347.329009:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575348.033053:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575348.033058:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575348.033062:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575348.352946:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575348.352957:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575348.352978:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575348.352985:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575348.352992:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575348.352997:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575348.353012:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575348.353017:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575348.353023:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575348.608879:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310708461, at 4310708480 00000100:00000010:0.0:1788575348.608895:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf240. 00000020:00000001:0.0:1788575349.057035:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575349.057038:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575349.057041:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575349.376970:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575349.376982:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575349.376991:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575349.377014:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575349.377033:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575349.377043:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575349.377061:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575349.377065:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575349.377076:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575349.888913:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575349.888927:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575349.888931:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575349.888935:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575349.888942:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575349.888949:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16037 00000400:00000200:0.0:1788575349.888960:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575349.888975:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575349.888978:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575349.888982:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.888986:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.888989:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.888991:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575349.889010:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575349.889012:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575349.889014:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575349.889017:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575349.889018:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575349.889020:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575349.889023:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889024:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.889026:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889027:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575349.889030:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575349.889033:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889036:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575349.889039:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575349.889042:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575349.889045:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.889047:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575349.889050:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575349.889056:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575349.889060:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575349.889064:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575349.889066:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.889069:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575349.889072:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575349.889078:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575349.889086:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575349.889091:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575349.889099:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516811392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575349.889109:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575349.889113:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575349.889118:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516811392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575349.889158:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575349.889161:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575349.889166:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575349.889168:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575349.889170:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575349.889174:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575349.889177:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16037 00000400:00000200:0.0:1788575349.889184:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575349.889197:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575349.889205:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575349.889214:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889219:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.889225:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889238:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575349.889247:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575349.889254:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575349.889264:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575349.889277:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575349.889305:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575349.889310:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575349.889315:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889317:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.889323:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889329:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575349.889333:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575349.889341:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889347:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575349.889373:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575349.889383:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575349.889391:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.889398:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575349.889410:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575349.889417:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.889423:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575349.889429:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575349.889438:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575349.889447:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575349.889451:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.889456:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575349.889460:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575349.889467:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575349.889477:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575349.889486:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575349.889502:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516811520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575349.889548:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575349.889554:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575349.889572:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516811520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575349.889599:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575349.889606:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575349.889617:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575349.889622:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575349.889626:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575349.889634:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575349.889647:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16037 00000400:00000200:0.0:1788575349.889657:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575349.889674:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575349.889682:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575349.889702:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889706:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.889709:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889714:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575349.889719:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575349.889725:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575349.889731:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575349.889735:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575349.889742:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575349.889747:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575349.889756:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889767:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.889772:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889787:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575349.889796:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575349.889801:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575349.889848:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575349.889853:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575349.889859:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575349.889867:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.889874:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575349.889879:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575349.889883:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.889889:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575349.889893:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575349.889897:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575349.889905:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575349.889907:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.889914:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575349.889917:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575349.889924:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575349.889932:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575349.889943:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575349.889952:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516811648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575349.889964:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575349.889969:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575349.889991:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516811648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575349.890064:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575349.890072:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575349.890089:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575349.890093:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575349.890096:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575349.890104:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575349.890113:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16037 00000400:00000200:0.0:1788575349.890129:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575349.890138:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575349.890143:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575349.890153:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.890162:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.890170:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.890180:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575349.890185:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575349.890188:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575349.890190:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575349.890192:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575349.890196:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575349.890199:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575349.890210:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.890219:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.890233:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.890242:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575349.890249:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575349.890263:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575349.890267:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575349.890272:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575349.890284:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575349.890307:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.890310:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575349.890313:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575349.890317:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.890319:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575349.890327:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575349.890333:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575349.890340:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575349.890342:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.890350:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575349.890354:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575349.890359:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575349.890368:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575349.890380:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575349.890390:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516811776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575349.890452:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575349.890456:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575349.890470:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516811776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575349.890501:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575349.890511:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575349.890516:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575349.890519:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575349.890531:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575349.890541:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575349.890553:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16037 00000400:00000200:0.0:1788575349.890568:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575349.890580:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575349.890588:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575349.890598:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.890609:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.890622:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.890632:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575349.890642:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575349.890647:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575349.890652:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575349.890659:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575349.890667:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575349.890676:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575349.890692:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.890699:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575349.890711:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575349.890724:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575349.890735:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575349.890748:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575349.890761:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575349.890765:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575349.890774:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575349.890785:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.890790:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575349.890796:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575349.890805:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.890839:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575349.890857:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575349.890866:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575349.890877:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575349.890884:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575349.890890:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575349.890896:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575349.890901:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575349.890913:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575349.890920:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575349.890931:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516811904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575349.890949:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575349.890952:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575349.890961:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516811904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575349.890975:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575349.890982:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16047) 00000100:00000001:0.0:1788575350.400945:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575350.400959:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575350.400963:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575350.400971:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575350.400981:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575350.400993:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575350.401008:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575350.401017:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575350.401027:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575350.401054:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575350.401065:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575350.401071:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575350.401075:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575350.401080:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575350.401085:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575350.401089:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575350.401091:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575350.401095:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575351.105055:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575351.105061:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575351.105065:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575351.424917:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575351.424923:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575351.424927:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575351.424931:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575351.424937:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575351.424942:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575351.424954:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575351.424963:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575351.424973:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575351.425010:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575351.425014:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575351.425018:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575351.425022:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575351.425027:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575351.425029:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575351.425031:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575351.425032:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575351.425034:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575352.448958:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575352.449165:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575352.449172:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575352.449175:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575352.449179:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575352.449181:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575352.449272:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575352.449275:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575352.449277:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575353.153109:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575353.153115:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575353.153119:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575353.472908:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575353.472922:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575353.472926:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575353.472933:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575353.472941:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575353.472969:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575353.472972:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575353.472974:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575353.472977:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575353.473038:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575353.473043:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575353.473046:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575353.473053:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575353.473060:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575353.473065:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575353.473080:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575353.473084:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575353.473087:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575353.728915:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310713581, at 4310713600 00000100:00000010:0.0:1788575353.728929:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf060. 00000020:00000001:0.0:1788575354.176986:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575354.176989:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575354.176991:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575354.496933:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575354.496942:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575354.496945:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575354.496957:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575354.496964:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575354.496969:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575354.496983:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575354.496988:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575354.496994:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575355.520931:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575355.520935:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575355.520940:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575355.520944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575355.520948:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575355.520954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575355.520971:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575355.520974:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575355.520977:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575356.224982:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575356.224999:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575356.225002:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575356.544893:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575356.544899:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575356.544904:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575356.544912:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575356.544919:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575356.544927:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575356.544937:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575356.544941:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575356.544945:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575357.568971:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575357.568983:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575357.568990:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575357.569002:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575357.569011:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575357.569020:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575357.569051:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575357.569055:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575357.569059:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575358.273022:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575358.273025:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575358.273028:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575358.592959:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575358.592968:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575358.592974:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575358.592985:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575358.592994:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575358.593000:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575358.593027:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575358.593030:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575358.593033:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575358.848963:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310718701, at 4310718720 00000100:00000010:0.0:1788575358.848986:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf660. 00000020:00000001:0.0:1788575359.297014:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575359.297020:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575359.297024:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575359.616899:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575359.616909:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575359.616913:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575359.616922:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575359.616930:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575359.616936:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575359.616951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575359.616955:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575359.616960:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575360.640947:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575360.640952:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575360.640955:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575360.640962:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575360.640967:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575360.640970:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575360.640983:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575360.640987:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575360.640992:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575361.344974:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575361.344978:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575361.344982:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575361.664910:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575361.664918:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575361.664924:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575361.664931:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575361.664937:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575361.664942:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575361.664946:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575361.664949:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575361.664951:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575361.665012:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575361.665016:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575361.665018:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575361.665020:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575361.665023:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575361.665025:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575361.665026:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575361.665028:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575361.665036:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575362.688896:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575362.688901:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575362.688905:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575362.688909:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575362.688916:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575362.688920:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575362.688934:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575362.688937:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575362.688941:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575363.393044:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575363.393047:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575363.393050:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575363.712901:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575363.712909:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575363.712912:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575363.712917:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575363.712921:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575363.712926:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575363.712929:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575363.712934:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575363.712939:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575363.712977:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575363.712980:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575363.712984:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575363.712987:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575363.712991:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575363.712994:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575363.712997:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575363.712999:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575363.713004:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575363.968906:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310723821, at 4310723840 00000100:00000010:0.0:1788575363.968924:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf0c0. 00000020:00000001:0.0:1788575364.417007:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575364.417011:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575364.417015:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575364.736911:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575364.736933:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575364.736938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575364.736943:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575364.736947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575364.736949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575364.736967:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575364.736972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575364.736975:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575365.760924:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575365.760934:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575365.760939:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575365.760947:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575365.760954:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575365.760959:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575365.760964:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575365.760968:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575365.760971:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575365.761072:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575365.761076:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575365.761079:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575365.761084:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575365.761086:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575365.761091:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575365.761096:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575365.761100:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575365.761104:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575366.465050:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575366.465054:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575366.465058:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575366.784991:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575366.784995:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575366.784996:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575366.784999:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575366.785002:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575366.785006:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575366.786511:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575366.786513:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575366.786521:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575367.808960:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575367.808968:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575367.808970:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575367.808976:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575367.808981:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575367.808986:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575367.808992:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575367.808998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575367.809006:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575368.321475:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575368.321483:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575368.321491:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575368.321499:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575365/real 1788575368] req@ffff8fc4ac9f4a80 x1875440516813312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575368.321516:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575368.321519:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575368.321523:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c454d0. 00000400:00000200:0.0:1788575368.321527:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc110 00000400:00000010:0.0:1788575368.321530:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc110. 00000100:00000001:0.0:1788575368.321533:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575368.321538:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4a80 x1875440516813312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575368.321550:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4a80 x1875440516813312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.321558:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575368.321561:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.321565:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575368.321568:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575368.321573:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4a80 x1875440516813312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.321582:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575368.321603:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4a80 x1875440516813312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.321641:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575368.321645:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.321648:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575368.321650:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.321653:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575368.321656:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575368.321661:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575368.321667:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575368.321671:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575368.321675:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4a80 x1875440516813312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575368.321698:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516813312:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575368.321705:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575368.321708:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575368.321711:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4a80 x1875440516813312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.321720:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575368.321724:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575368.321730:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575368.321737:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575368.321740:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575368.321743:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575368.321746:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575368.321749:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34400. 02000000:00000001:0.0:1788575368.321758:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575368.321761:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575368.321763:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4a80. 00000100:00000001:0.0:1788575368.321768:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575368.321770:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.321773:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575368.321787:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575368.321789:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575368.321791:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575368.321796:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575365/real 1788575368] req@ffff8fc4ac9f6d80 x1875440516813696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575368.321806:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575368.321861:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575368.321865:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c45c08. 00000400:00000200:0.0:1788575368.321869:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc6e8 00000400:00000010:0.0:1788575368.321871:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc6e8. 00000100:00000001:0.0:1788575368.321874:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575368.321878:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6d80 x1875440516813696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575368.321886:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6d80 x1875440516813696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.321893:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575368.321895:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.321897:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575368.321899:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575368.321903:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6d80 x1875440516813696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.321910:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575368.321914:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6d80 x1875440516813696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.321921:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575368.321924:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.321926:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575368.321928:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.321930:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575368.321934:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575368.321937:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575368.321948:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575368.321951:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575368.321956:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6d80 x1875440516813696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575368.321964:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516813696:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575368.321969:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575368.321973:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575365/real 1788575368] req@ffff8fc4ac9f4e00 x1875440516813824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575368.321983:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575368.321985:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575368.321987:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c45ec8. 00000400:00000200:0.0:1788575368.321990:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcaa0 00000400:00000010:0.0:1788575368.321992:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcaa0. 00000100:00000001:0.0:1788575368.321995:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575368.321999:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4e00 x1875440516813824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575368.322023:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4e00 x1875440516813824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322030:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575368.322032:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322034:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575368.322035:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575368.322039:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4e00 x1875440516813824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322046:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575368.322050:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4e00 x1875440516813824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322070:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575368.322073:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322074:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575368.322077:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322079:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575368.322082:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575368.322085:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575368.322088:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575368.322092:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575368.322096:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4e00 x1875440516813824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575368.322103:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516813824:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575368.322107:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575368.322111:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575365/real 1788575368] req@ffff8fc4ac9f5880 x1875440516813440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575368.322119:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575368.322121:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575368.322123:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c45108. 00000400:00000200:0.0:1788575368.322126:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcf68 00000400:00000010:0.0:1788575368.322128:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcf68. 00000100:00000001:0.0:1788575368.322131:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575368.322134:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5880 x1875440516813440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575368.322141:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5880 x1875440516813440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322148:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575368.322158:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322160:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575368.322161:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575368.322165:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5880 x1875440516813440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322172:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575368.322176:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5880 x1875440516813440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322182:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575368.322184:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322186:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575368.322188:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322190:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575368.322192:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575368.322195:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575368.322199:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575368.322202:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575368.322207:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5880 x1875440516813440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575368.322215:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516813440:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575368.322220:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575368.322225:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575365/real 1788575368] req@ffff8fc4ac9f7b80 x1875440516813568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575368.322234:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575368.322235:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575368.322238:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c45a50. 00000400:00000200:0.0:1788575368.322241:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc330 00000400:00000010:0.0:1788575368.322244:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc330. 00000100:00000001:0.0:1788575368.322252:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575368.322256:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7b80 x1875440516813568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575368.322264:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7b80 x1875440516813568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322270:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575368.322272:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322275:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575368.322276:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575368.322280:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7b80 x1875440516813568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322287:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575368.322291:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7b80 x1875440516813568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322298:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575368.322300:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322302:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575368.322303:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322305:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575368.322310:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575368.322312:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575368.322317:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575368.322319:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575368.322324:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7b80 x1875440516813568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575368.322332:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516813568:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575368.322337:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322345:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575368.322349:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6d80 x1875440516813696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322357:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575368.322360:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575368.322362:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575368.322367:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575368.322369:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575368.322371:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575368.322373:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575368.322375:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788575368.322378:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575368.322380:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575368.322382:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6d80. 00000100:00000001:0.0:1788575368.322385:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575368.322387:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322389:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575368.322392:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4e00 x1875440516813824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322399:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575368.322400:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575368.322402:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575368.322404:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575368.322406:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575368.322407:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575368.322410:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575368.322412:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36c00. 02000000:00000001:0.0:1788575368.322414:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575368.322416:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575368.322417:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4e00. 00000100:00000001:0.0:1788575368.322420:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575368.322421:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322422:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575368.322426:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5880 x1875440516813440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322444:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575368.322446:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575368.322447:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575368.322450:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575368.322452:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575368.322454:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575368.322456:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575368.322457:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35800. 02000000:00000001:0.0:1788575368.322460:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575368.322461:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575368.322463:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5880. 00000100:00000001:0.0:1788575368.322465:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575368.322467:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322469:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575368.322473:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7b80 x1875440516813568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575368.322486:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575368.322488:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575368.322489:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575368.322492:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575368.322493:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575368.322495:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575368.322497:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575368.322499:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36400. 02000000:00000001:0.0:1788575368.322501:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575368.322502:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575368.322503:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7b80. 00000100:00000001:0.0:1788575368.322506:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575368.322508:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322510:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.322523:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575368.322526:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575368.322530:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575368.322531:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575368.513096:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575368.513128:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575368.513132:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575368.832984:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575368.832995:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575368.833009:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575368.833015:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575368.833021:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575368.833027:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.833038:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575368.833042:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.833049:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575368.833090:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575368.833094:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575368.833097:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575368.833100:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575368.833104:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575368.833109:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.833118:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575368.833120:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575368.833128:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575369.088909:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310728941, at 4310728960 00000100:00000010:0.0:1788575369.088935:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf840. 00000020:00000001:0.0:1788575369.537094:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575369.537099:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575369.537103:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575370.369045:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575370.369055:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575370.369059:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575370.369063:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575370.369067:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575370.369075:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16057 00000400:00000200:0.0:1788575370.369083:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575370.369089:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575370.369092:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575370.369096:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369121:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575370.369124:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369127:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575370.369131:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575370.369133:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575370.369136:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575370.369139:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575370.369142:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575370.369144:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575370.369147:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369149:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575370.369151:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369153:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575370.369156:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575370.369160:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369163:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575370.369166:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575370.369169:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575370.369172:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.369175:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575370.369177:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575370.369188:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575370.369192:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575370.369196:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575370.369198:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.369202:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575370.369205:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575370.369210:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575370.369217:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575370.369224:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575370.369234:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516813952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575370.369257:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575370.369262:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575370.369266:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516813952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575370.369287:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575370.369291:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575370.369294:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575370.369296:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575370.369298:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575370.369301:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575370.369304:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16057 00000400:00000200:0.0:1788575370.369312:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575370.369316:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575370.369319:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575370.369322:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369328:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575370.369331:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369336:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575370.369341:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575370.369344:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575370.369348:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575370.369352:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575370.369358:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575370.369363:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575370.369387:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369392:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575370.369399:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369403:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575370.369414:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575370.369421:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369426:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575370.369431:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575370.369439:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575370.369447:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.369449:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575370.369455:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575370.369460:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.369462:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575370.369466:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575370.369471:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575370.369474:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575370.369478:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.369482:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575370.369484:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575370.369487:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575370.369494:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575370.369498:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575370.369505:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516814080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575370.369523:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575370.369528:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575370.369536:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516814080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575370.369552:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575370.369558:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575370.369568:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575370.369570:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575370.369574:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575370.369579:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575370.369592:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16057 00000400:00000200:0.0:1788575370.369602:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575370.369610:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575370.369614:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575370.369621:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369626:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575370.369633:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369639:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575370.369643:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575370.369647:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575370.369651:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575370.369656:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575370.369662:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575370.369667:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575370.369672:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369676:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575370.369683:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369689:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575370.369694:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575370.369701:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369705:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575370.369708:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575370.369714:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575370.369719:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.369725:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575370.369730:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575370.369735:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.369738:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575370.369744:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575370.369751:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575370.369759:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575370.369767:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.369772:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575370.369775:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575370.369779:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575370.369788:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575370.369795:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575370.369805:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516814208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575370.369852:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575370.369854:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575370.369858:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516814208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575370.369866:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575370.369869:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575370.369873:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575370.369875:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575370.369877:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575370.369880:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575370.369883:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16057 00000400:00000200:0.0:1788575370.369887:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575370.369890:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575370.369892:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575370.369895:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369898:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575370.369900:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369902:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575370.369904:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575370.369906:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575370.369908:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575370.369911:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575370.369913:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575370.369922:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575370.369926:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369931:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575370.369938:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369944:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575370.369950:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575370.369956:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575370.369962:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575370.369967:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575370.369972:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575370.369990:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.369997:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575370.370002:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575370.370006:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.370009:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575370.370016:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575370.370023:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575370.370029:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575370.370034:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.370039:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575370.370041:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575370.370043:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575370.370049:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575370.370054:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575370.370063:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516814336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575370.370077:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575370.370081:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575370.370090:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516814336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575370.370106:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575370.370120:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575370.370130:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575370.370134:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575370.370138:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575370.370145:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575370.370151:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16057 00000400:00000200:0.0:1788575370.370158:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575370.370165:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575370.370169:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575370.370174:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.370180:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575370.370185:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.370191:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575370.370197:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575370.370200:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575370.370204:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575370.370208:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575370.370213:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575370.370218:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575370.370228:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.370233:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575370.370237:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575370.370242:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575370.370247:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575370.370253:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575370.370258:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575370.370262:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575370.370267:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575370.370273:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.370278:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575370.370289:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575370.370295:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.370300:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575370.370306:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575370.370311:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575370.370317:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575370.370321:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575370.370324:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575370.370329:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575370.370333:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575370.370340:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575370.370346:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575370.370357:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516814464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575370.370371:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575370.370375:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575370.370384:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516814464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575370.370401:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575370.370410:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16067) 00000100:00000001:0.0:1788575370.880989:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575370.881001:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575370.881014:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575370.881034:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575370.881049:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575370.881057:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575370.881077:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575370.881087:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575370.881097:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575371.585071:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575371.585076:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575371.585080:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575371.904980:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575371.904992:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575371.905013:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575371.905022:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575371.905033:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575371.905038:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575371.905067:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575371.905071:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575371.905076:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575372.928900:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575372.928910:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575372.928915:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575372.928923:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575372.928931:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575372.928948:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575372.928956:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575372.928961:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575372.928966:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575373.633060:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575373.633063:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575373.633065:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575373.952958:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575373.952965:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575373.952971:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575373.952981:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575373.952987:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575373.952993:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575373.953001:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575373.953006:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575373.953010:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575374.208905:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310734062, at 4310734080 00000100:00000010:0.0:1788575374.208933:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfe40. 00000020:00000001:0.0:1788575374.657006:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575374.657009:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575374.657012:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575374.976920:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575374.976926:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575374.976930:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575374.976936:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575374.976956:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575374.976960:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575374.976963:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575374.976965:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575374.976967:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575376.000916:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575376.000925:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575376.000930:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575376.000936:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575376.000943:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575376.000949:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575376.000963:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575376.000966:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575376.000972:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575376.704969:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575376.704972:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575376.704974:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575377.024924:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575377.024930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575377.024935:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575377.024941:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575377.024944:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575377.024948:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575377.024960:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575377.024964:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575377.024971:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575377.537115:0:2391:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788575377.537120:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575377.537134:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575377.537140:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575377.537143:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575378.048959:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575378.048966:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575378.048972:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575378.048979:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575378.048985:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575378.048990:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575378.049007:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575378.049019:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575378.049025:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575378.752963:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575378.752968:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575378.752971:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575379.072932:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575379.072942:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575379.072947:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575379.072954:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575379.072961:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575379.072967:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575379.072978:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575379.072982:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575379.072985:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575379.328904:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310739181, at 4310739200 00000100:00000010:0.0:1788575379.328919:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfcc0. 00000020:00000001:0.0:1788575379.776993:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575379.776997:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575379.777000:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575380.097086:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575380.097089:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575380.097091:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575380.097094:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575380.097097:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575380.097099:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.097101:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575380.097103:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.097105:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.609138:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575380.609142:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575380.609144:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575380.609153:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516815232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.609168:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575380.609173:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.609181:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575380.609201:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575380.609207:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575380.609217:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516815232:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575380.609227:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575380.609234:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575380.609239:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.609245:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575380.609253:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575380.609259:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575380.609274:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575380.609283:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575380.609294:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575380.609305:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516815232, portal 25 00000100:00000001:0.0:1788575380.609313:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575380.609317:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575380.609334:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516815232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.609353:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575380.609360:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575380.609367:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575380.609373:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516815232, offset 0 00000400:00000200:0.0:1788575380.609384:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575380.609394:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575380.609411:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575380.609419:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575380.609424:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575380.609427:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575380.609430:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575380.609435:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548021:548021:256:4294967295] 192.168.202.128@tcp LPNI seq info [547944:547944:8:4294967295] 00000400:00000200:0.0:1788575380.609443:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575380.609459:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575380.609471:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575380.609486:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575380.609501:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575380.609509:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788575380.609536:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575380.609538:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575380.609542:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575380.609548:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575380.609556:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575380.609561:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575380.609567:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575380.609576:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516815232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.609609:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575380.609618:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516815232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.609636:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.609640:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575380.609648:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575380.609656:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575380.609660:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310744301 00000100:00000010:0.0:1788575380.609668:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575380.609676:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575380.609683:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310744301 00000100:00000010:0.0:1788575380.609689:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b500c0. 00000100:00000001:0.0:1788575380.609693:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575380.609698:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575380.609703:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.609707:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575380.609714:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575380/real 1788575380] req@ffff8fc4901a4e00 x1875440516815232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575380.609763:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575380.609766:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575380.609770:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575380.609778:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575380.609782:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575380.609788:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575380.609795:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516815232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575380.609842:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516815232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.609848:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575380.609851:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.609855:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575380.609858:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575380.609864:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516815232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.609875:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575380.609883:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516815232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.609896:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575380.609901:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.609906:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575380.609907:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.609911:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575380.609914:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575380.609918:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575380.609926:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575380.609932:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575380.609940:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516815232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575380.609965:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516815232:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575380.609971:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.609974:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575380.609980:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516815232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.609996:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575380.610000:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575380.610003:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575380.610006:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575380.610009:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575380.610011:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575380.610013:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575380.610014:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575380.610018:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575380.610020:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575380.610022:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575380.610028:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575380.610030:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610032:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610038:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575380.610040:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.610043:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575380.610045:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575380.610047:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575380.610051:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516815360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610057:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575380.610058:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.610060:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575380.610062:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575380.610064:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575380.610066:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516815360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575380.610073:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575380.610075:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575380.610076:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.610078:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575380.610080:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575380.610081:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575380.610085:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575380.610086:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575380.610089:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575380.610091:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516815360, portal 10 00000100:00000001:0.0:1788575380.610093:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575380.610094:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575380.610098:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516815360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610103:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575380.610104:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575380.610105:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575380.610107:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516815360, offset 0 00000400:00000200:0.0:1788575380.610110:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575380.610112:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575380.610115:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575380.610117:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575380.610119:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575380.610121:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575380.610122:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575380.610125:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548022:548022:256:4294967295] 192.168.202.128@tcp LPNI seq info [547945:547945:8:4294967295] 00000400:00000200:0.0:1788575380.610128:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575380.610132:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575380.610135:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575380.610137:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575380.610140:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575380.610145:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788575380.610152:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575380.610154:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575380.610156:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575380.610158:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575380.610161:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575380.610162:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575380.610164:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575380.610166:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516815360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610172:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575380.610174:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516815360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610178:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.610179:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575380.610181:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575380.610184:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575380.610186:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575380.610188:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310740499 00000100:00000001:0.0:1788575380.610190:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310744301 00000100:00000001:0.0:1788575380.610192:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310740499 00000100:00000001:0.0:1788575380.610193:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310744301 00000100:00000001:0.0:1788575380.610194:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575380.610196:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.610197:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575380.610199:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575380/real 1788575380] req@ffff8fc4901a6300 x1875440516815360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575380.610204:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575380.610205:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575380.610209:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575380.610211:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575380.610212:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575380.610214:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575380.610216:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516815360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575380.610221:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516815360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610225:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575380.610226:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610227:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575380.610228:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575380.610230:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516815360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610235:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575380.610237:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516815360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610241:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575380.610243:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610244:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575380.610245:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610246:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575380.610247:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575380.610249:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575380.610251:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575380.610252:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575380.610255:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516815360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575380.610259:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516815360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575380.610265:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575380.610267:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516815488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610272:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575380.610273:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.610274:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575380.610275:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575380.610276:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575380.610278:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516815488:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575380.610280:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575380.610282:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575380.610283:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.610284:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575380.610286:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575380.610287:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575380.610290:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575380.610291:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575380.610293:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575380.610295:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516815488, portal 10 00000100:00000001:0.0:1788575380.610297:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575380.610298:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575380.610302:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516815488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610306:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575380.610307:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575380.610309:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575380.610310:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516815488, offset 0 00000400:00000200:0.0:1788575380.610313:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575380.610315:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575380.610317:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575380.610319:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575380.610324:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575380.610326:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575380.610327:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575380.610330:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548023:548023:256:4294967295] 192.168.202.128@tcp LPNI seq info [547946:547946:8:4294967295] 00000400:00000200:0.0:1788575380.610332:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575380.610337:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575380.610340:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575380.610342:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575380.610344:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575380.610346:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788575380.610357:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575380.610358:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575380.610360:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575380.610362:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575380.610365:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575380.610366:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575380.610367:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575380.610370:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516815488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610375:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575380.610377:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516815488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610381:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.610382:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575380.610384:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575380.610386:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575380.610388:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575380.610389:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310740506 00000100:00000001:0.0:1788575380.610391:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310744301 00000100:00000001:0.0:1788575380.610392:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310740506 00000100:00000001:0.0:1788575380.610396:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310744301 00000100:00000001:0.0:1788575380.610398:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575380.610399:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.610401:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575380.610403:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575380/real 1788575380] req@ffff8fc4901a7480 x1875440516815488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575380.610409:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575380.610410:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575380.610411:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575380.610412:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575380.610413:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575380.610415:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575380.610417:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516815488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575380.610422:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516815488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610426:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575380.610427:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610428:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575380.610429:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575380.610431:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516815488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610435:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575380.610438:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516815488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610442:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575380.610443:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610444:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575380.610445:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610448:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575380.610450:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575380.610452:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575380.610454:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575380.610456:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575380.610459:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516815488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575380.610464:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516815488:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575380.610467:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575380.610471:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516815616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610478:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575380.610479:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.610481:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575380.610483:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575380.610485:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575380.610488:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516815616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575380.610491:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575380.610493:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575380.610495:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.610496:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575380.610500:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575380.610502:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575380.610506:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575380.610509:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575380.610512:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575380.610515:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516815616, portal 4 00000100:00000001:0.0:1788575380.610518:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575380.610519:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575380.610524:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516815616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610533:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575380.610535:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575380.610537:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575380.610538:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516815616, offset 0 00000400:00000200:0.0:1788575380.610541:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575380.610544:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575380.610546:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575380.610548:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575380.610550:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575380.610552:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575380.610553:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575380.610555:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548024:548024:256:4294967295] 192.168.202.128@tcp LPNI seq info [547947:547947:8:4294967295] 00000400:00000200:0.0:1788575380.610558:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575380.610562:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575380.610565:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575380.610567:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575380.610569:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575380.610571:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788575380.610582:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575380.610584:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575380.610587:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575380.610590:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575380.610593:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575380.610595:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575380.610598:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575380.610602:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516815616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610611:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575380.610614:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516815616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610630:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.610631:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575380.610635:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575380.610637:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575380.610640:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575380.610641:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310740646 00000100:00000001:0.0:1788575380.610643:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310744301 00000100:00000001:0.0:1788575380.610645:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310740646 00000100:00000001:0.0:1788575380.610646:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310744301 00000100:00000001:0.0:1788575380.610647:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575380.610648:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.610650:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575380.610652:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575380/real 1788575380] req@ffff8fc4901a4700 x1875440516815616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575380.610657:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575380.610658:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575380.610659:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575380.610661:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575380.610662:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575380.610664:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575380.610665:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516815616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575380.610671:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516815616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610675:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575380.610675:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610677:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575380.610677:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575380.610680:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516815616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610704:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575380.610708:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516815616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610713:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575380.610715:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610716:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575380.610717:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610718:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575380.610720:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575380.610721:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575380.610723:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575380.610725:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575380.610728:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516815616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575380.610734:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516815616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575380.610737:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575380.610739:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516815744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610743:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575380.610744:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.610746:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575380.610747:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575380.610749:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575380.610751:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516815744:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575380.610753:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575380.610754:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575380.610758:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575380.610760:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575380.610762:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575380.610763:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575380.610766:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575380.610767:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575380.610770:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575380.610772:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516815744, portal 4 00000100:00000001:0.0:1788575380.610773:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575380.610774:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575380.610778:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516815744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610783:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575380.610784:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575380.610785:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575380.610787:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516815744, offset 0 00000400:00000200:0.0:1788575380.610789:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575380.610792:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575380.610794:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575380.610796:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575380.610798:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575380.610799:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575380.610801:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575380.610803:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548025:548025:256:4294967295] 192.168.202.128@tcp LPNI seq info [547948:547948:8:4294967295] 00000400:00000200:0.0:1788575380.610806:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575380.610840:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575380.610844:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575380.610847:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575380.610850:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575380.610852:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 47 secs later) 00000800:00000100:0.0:1788575380.610860:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575380.610867:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575380.610870:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575380.610872:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575380.610875:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575380.610876:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575380.610878:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575380.610881:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516815744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610886:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575380.610888:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516815744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610893:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.610894:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575380.610896:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575380.610898:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575380.610901:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575380.610902:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310740649 00000100:00000001:0.0:1788575380.610904:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310744301 00000100:00000001:0.0:1788575380.610905:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310740649 00000100:00000001:0.0:1788575380.610906:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310744301 00000100:00000001:0.0:1788575380.610908:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575380.610909:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.610910:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575380.610912:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575380/real 1788575380] req@ffff8fc4901a5880 x1875440516815744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575380.610917:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575380.610918:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575380.610919:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575380.610921:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575380.610922:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575380.610927:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575380.610929:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516815744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575380.610936:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516815744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610941:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575380.610942:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610944:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575380.610945:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575380.610948:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516815744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610952:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575380.610954:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516815744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610958:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575380.610960:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610961:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575380.610961:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610963:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575380.610964:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575380.610966:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575380.610967:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575380.610969:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575380.610971:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516815744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575380.610976:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516815744:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575380.610978:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.610980:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575380.610985:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516815360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.610989:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575380.610991:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575380.610992:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575380.610994:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575380.610995:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575380.610996:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575380.610997:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575380.610998:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575380.611000:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575380.611001:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575380.611002:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575380.611005:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575380.611006:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.611008:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575380.611010:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516815488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.611014:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575380.611015:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575380.611016:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575380.611017:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575380.611018:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575380.611018:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575380.611019:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575380.611020:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575380.611022:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575380.611023:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575380.611023:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575380.611025:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575380.611026:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.611027:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575380.611029:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516815616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.611033:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575380.611036:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575380.611037:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575380.611039:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575380.611040:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575380.611040:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575380.611042:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575380.611043:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575380.611044:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575380.611045:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575380.611045:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575380.611047:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575380.611048:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.611049:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575380.611051:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516815744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575380.611055:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575380.611055:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575380.611056:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575380.611057:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575380.611058:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575380.611059:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575380.611060:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575380.611061:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575380.611062:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575380.611062:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575380.611063:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575380.611065:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575380.611065:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.611067:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575380.611069:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575380.611070:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575380.611071:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575380.611072:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575381.120928:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575381.120935:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575381.120940:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575381.120953:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575381.120975:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575381.120979:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575381.120993:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575381.120997:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575381.121001:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575381.825020:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575381.825023:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575381.825026:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575382.144920:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575382.144933:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575382.144935:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575382.144940:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575382.144946:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575382.144949:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575382.144951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575382.144954:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575382.144959:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575383.168931:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575383.168941:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575383.168946:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575383.168954:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575383.168962:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575383.168971:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575383.168985:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575383.168989:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575383.168994:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575383.873012:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575383.873015:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575383.873017:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575384.192903:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575384.192912:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575384.192914:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575384.192922:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575384.192929:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575384.192936:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575384.192943:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575384.192948:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575384.192955:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575384.448874:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310744301, at 4310744320 00000100:00000010:0.0:1788575384.448892:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575384.897044:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575384.897048:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575384.897051:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575385.216918:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575385.216925:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575385.216931:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575385.216939:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575385.216945:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575385.216948:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.216951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575385.216955:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.216959:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.729140:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575385.729146:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575385.729149:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575385.729158:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6680 x1875440516815872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.729166:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575385.729169:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575385.729175:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575385.729177:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575385.729179:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575385.729183:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516815872:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575385.729190:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575385.729195:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575385.729198:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575385.729209:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575385.729216:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575385.729221:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575385.729230:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575385.729236:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575385.729246:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575385.729256:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516815872, portal 25 00000100:00000001:0.0:1788575385.729271:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575385.729273:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605330560 : -123400104221056 : ffff8fc4ac9f6680) 00000100:00000040:0.0:1788575385.729283:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6680 x1875440516815872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.729298:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575385.729304:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575385.729308:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575385.729313:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516815872, offset 0 00000400:00000200:0.0:1788575385.729320:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575385.729331:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575385.729340:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575385.729346:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575385.729354:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575385.729363:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575385.729372:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575385.729383:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548026:548026:256:4294967295] 192.168.202.128@tcp LPNI seq info [547949:547949:8:4294967295] 00000400:00000200:0.0:1788575385.729394:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575385.729414:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575385.729430:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575385.729441:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575385.729454:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575385.729460:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788575385.729493:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575385.729498:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575385.729508:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575385.729517:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575385.729526:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575385.729530:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575385.729536:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575385.729544:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6680 x1875440516815872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.729581:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575385.729587:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6680 x1875440516815872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.729603:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.729608:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575385.729615:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575385.729620:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575385.729625:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310749421 00000100:00000010:0.0:1788575385.729631:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575385.729640:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575385.729649:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310749421 00000100:00000010:0.0:1788575385.729655:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50ba0. 00000100:00000001:0.0:1788575385.729660:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575385.729666:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575385.729670:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.729675:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575385.729692:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575385/real 1788575385] req@ffff8fc4ac9f6680 x1875440516815872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575385.729714:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575385.729719:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575385.729724:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575385.729730:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575385.729735:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575385.729755:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575385.729765:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6680 x1875440516815872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575385.729788:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6680 x1875440516815872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.729807:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575385.729856:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.729861:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575385.729864:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575385.729872:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6680 x1875440516815872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.729887:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575385.729893:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6680 x1875440516815872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.729898:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575385.729903:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.729906:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575385.729907:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.729912:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575385.729916:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575385.729920:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575385.729928:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575385.729934:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575385.729940:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6680 x1875440516815872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575385.729951:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516815872:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575385.729957:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.729961:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575385.729968:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6680 x1875440516815872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.729982:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575385.729986:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575385.729989:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575385.729994:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575385.729997:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575385.730005:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575385.730007:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575385.730009:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36c00. 02000000:00000001:0.0:1788575385.730013:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575385.730014:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575385.730016:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6680. 00000100:00000001:0.0:1788575385.730018:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575385.730019:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730021:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730026:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575385.730028:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.730029:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575385.730031:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575385.730032:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575385.730035:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4000 x1875440516816000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730040:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575385.730041:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575385.730043:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575385.730044:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575385.730046:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575385.730047:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816000:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575385.730049:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575385.730051:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575385.730052:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575385.730053:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575385.730055:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575385.730056:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575385.730060:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575385.730062:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575385.730064:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575385.730066:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516816000, portal 10 00000100:00000001:0.0:1788575385.730068:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575385.730069:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605320704 : -123400104230912 : ffff8fc4ac9f4000) 00000100:00000040:0.0:1788575385.730074:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4000 x1875440516816000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730082:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575385.730083:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575385.730085:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575385.730087:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516816000, offset 0 00000400:00000200:0.0:1788575385.730089:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575385.730092:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575385.730095:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575385.730098:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575385.730099:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575385.730101:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575385.730103:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575385.730105:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548027:548027:256:4294967295] 192.168.202.128@tcp LPNI seq info [547950:547950:8:4294967295] 00000400:00000200:0.0:1788575385.730108:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575385.730113:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575385.730116:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575385.730118:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575385.730121:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575385.730123:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788575385.730131:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575385.730132:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575385.730134:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575385.730137:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575385.730139:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575385.730140:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575385.730142:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575385.730144:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4000 x1875440516816000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730150:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575385.730152:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4000 x1875440516816000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730164:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.730165:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575385.730168:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575385.730170:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575385.730173:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575385.730174:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310745619 00000100:00000001:0.0:1788575385.730176:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310749421 00000100:00000001:0.0:1788575385.730178:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310745619 00000100:00000001:0.0:1788575385.730179:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310749421 00000100:00000001:0.0:1788575385.730181:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575385.730182:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.730183:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575385.730186:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575385/real 1788575385] req@ffff8fc4ac9f4000 x1875440516816000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575385.730191:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575385.730192:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575385.730194:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575385.730195:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575385.730196:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575385.730198:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575385.730200:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4000 x1875440516816000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575385.730205:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4000 x1875440516816000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730209:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575385.730210:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730211:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575385.730212:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575385.730214:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4000 x1875440516816000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730221:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575385.730224:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4000 x1875440516816000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730229:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575385.730230:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730231:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575385.730232:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730233:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575385.730235:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575385.730236:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575385.730238:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575385.730240:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575385.730243:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4000 x1875440516816000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575385.730247:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816000:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575385.730250:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575385.730252:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7100 x1875440516816128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730256:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575385.730256:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575385.730258:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575385.730259:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575385.730260:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575385.730262:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816128:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575385.730264:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575385.730265:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575385.730269:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575385.730270:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575385.730272:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575385.730274:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575385.730276:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575385.730278:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575385.730280:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575385.730282:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516816128, portal 10 00000100:00000001:0.0:1788575385.730283:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575385.730284:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605333248 : -123400104218368 : ffff8fc4ac9f7100) 00000100:00000040:0.0:1788575385.730289:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7100 x1875440516816128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730293:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575385.730295:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575385.730296:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575385.730298:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516816128, offset 0 00000400:00000200:0.0:1788575385.730300:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575385.730303:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575385.730305:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575385.730307:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575385.730308:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575385.730310:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575385.730311:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575385.730314:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548028:548028:256:4294967295] 192.168.202.128@tcp LPNI seq info [547951:547951:8:4294967295] 00000400:00000200:0.0:1788575385.730316:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575385.730320:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575385.730323:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575385.730325:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575385.730327:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575385.730329:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788575385.730336:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575385.730342:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575385.730344:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575385.730346:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575385.730349:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575385.730350:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575385.730353:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575385.730356:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7100 x1875440516816128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730365:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575385.730368:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7100 x1875440516816128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730375:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.730377:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575385.730380:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575385.730383:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575385.730387:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575385.730389:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310745626 00000100:00000001:0.0:1788575385.730392:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310749421 00000100:00000001:0.0:1788575385.730394:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310745626 00000100:00000001:0.0:1788575385.730397:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310749421 00000100:00000001:0.0:1788575385.730399:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575385.730401:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.730403:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575385.730407:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575385/real 1788575385] req@ffff8fc4ac9f7100 x1875440516816128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575385.730415:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575385.730417:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575385.730419:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575385.730421:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575385.730423:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575385.730432:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575385.730436:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7100 x1875440516816128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575385.730446:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7100 x1875440516816128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730454:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575385.730456:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730458:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575385.730459:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575385.730463:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7100 x1875440516816128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730470:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575385.730474:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7100 x1875440516816128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730481:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575385.730484:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730485:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575385.730487:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730489:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575385.730492:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575385.730494:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575385.730498:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575385.730500:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575385.730505:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7100 x1875440516816128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575385.730512:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816128:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575385.730515:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575385.730522:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7800 x1875440516816256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730527:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575385.730528:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575385.730530:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575385.730531:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575385.730532:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575385.730534:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816256:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575385.730536:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575385.730538:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575385.730539:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575385.730540:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575385.730542:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575385.730544:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575385.730546:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575385.730548:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575385.730551:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575385.730553:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516816256, portal 4 00000100:00000001:0.0:1788575385.730554:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575385.730555:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335040 : -123400104216576 : ffff8fc4ac9f7800) 00000100:00000040:0.0:1788575385.730559:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7800 x1875440516816256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730564:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575385.730565:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575385.730567:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575385.730568:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516816256, offset 0 00000400:00000200:0.0:1788575385.730571:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575385.730573:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575385.730575:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575385.730577:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575385.730579:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575385.730581:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575385.730582:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575385.730588:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548029:548029:256:4294967295] 192.168.202.128@tcp LPNI seq info [547952:547952:8:4294967295] 00000400:00000200:0.0:1788575385.730591:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575385.730595:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575385.730598:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575385.730600:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575385.730603:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575385.730605:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788575385.730612:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575385.730613:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575385.730615:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575385.730618:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575385.730620:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575385.730621:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575385.730623:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575385.730625:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7800 x1875440516816256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730631:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575385.730633:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7800 x1875440516816256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730637:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.730638:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575385.730640:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575385.730642:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575385.730645:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575385.730646:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310745766 00000100:00000001:0.0:1788575385.730648:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310749421 00000100:00000001:0.0:1788575385.730649:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310745766 00000100:00000001:0.0:1788575385.730651:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310749421 00000100:00000001:0.0:1788575385.730652:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575385.730656:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.730658:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575385.730661:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575385/real 1788575385] req@ffff8fc4ac9f7800 x1875440516816256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575385.730666:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575385.730667:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575385.730669:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575385.730670:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575385.730671:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575385.730672:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575385.730674:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7800 x1875440516816256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575385.730679:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7800 x1875440516816256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730684:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575385.730685:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730687:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575385.730688:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575385.730692:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7800 x1875440516816256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730699:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575385.730702:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7800 x1875440516816256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730710:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575385.730712:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730714:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575385.730715:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730717:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575385.730719:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575385.730722:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575385.730731:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575385.730734:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575385.730738:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7800 x1875440516816256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575385.730763:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816256:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575385.730768:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575385.730772:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5180 x1875440516816384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730779:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575385.730780:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575385.730783:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575385.730785:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575385.730787:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575385.730790:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816384:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575385.730794:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575385.730795:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575385.730797:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575385.730799:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575385.730802:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575385.730803:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575385.730806:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575385.730822:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575385.730825:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575385.730828:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516816384, portal 4 00000100:00000001:0.0:1788575385.730829:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575385.730831:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605325184 : -123400104226432 : ffff8fc4ac9f5180) 00000100:00000040:0.0:1788575385.730835:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5180 x1875440516816384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730839:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575385.730845:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575385.730847:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575385.730848:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516816384, offset 0 00000400:00000200:0.0:1788575385.730851:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575385.730854:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575385.730857:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575385.730859:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575385.730861:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575385.730862:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575385.730864:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575385.730866:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548030:548030:256:4294967295] 192.168.202.128@tcp LPNI seq info [547953:547953:8:4294967295] 00000400:00000200:0.0:1788575385.730869:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575385.730873:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575385.730876:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575385.730878:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575385.730881:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575385.730883:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1788575385.730890:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575385.730891:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575385.730894:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575385.730896:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575385.730898:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575385.730899:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575385.730901:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575385.730903:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5180 x1875440516816384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730909:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575385.730911:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5180 x1875440516816384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730915:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.730917:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575385.730924:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575385.730926:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575385.730928:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575385.730930:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310745769 00000100:00000001:0.0:1788575385.730931:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310749421 00000100:00000001:0.0:1788575385.730933:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310745769 00000100:00000001:0.0:1788575385.730934:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310749421 00000100:00000001:0.0:1788575385.730936:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575385.730937:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.730938:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575385.730941:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575385/real 1788575385] req@ffff8fc4ac9f5180 x1875440516816384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575385.730946:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575385.730947:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575385.730948:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575385.730949:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575385.730951:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575385.730952:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575385.730954:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516816384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575385.730959:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516816384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730963:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575385.730964:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730965:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575385.730966:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575385.730968:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516816384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730972:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575385.730975:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516816384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.730982:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575385.730984:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730985:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575385.730986:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.730987:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575385.730988:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575385.730990:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575385.730992:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575385.730993:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575385.730996:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516816384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575385.731001:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816384:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575385.731004:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.731005:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575385.731007:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4000 x1875440516816000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.731011:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575385.731012:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575385.731013:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575385.731015:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575385.731016:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575385.731017:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575385.731018:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575385.731019:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788575385.731021:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575385.731022:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575385.731023:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4000. 00000100:00000001:0.0:1788575385.731025:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575385.731026:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.731030:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575385.731032:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7100 x1875440516816128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.731037:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575385.731037:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575385.731038:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575385.731040:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575385.731041:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575385.731041:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575385.731042:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575385.731043:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34400. 02000000:00000001:0.0:1788575385.731045:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575385.731045:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575385.731046:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7100. 00000100:00000001:0.0:1788575385.731048:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575385.731048:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.731049:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575385.731051:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7800 x1875440516816256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.731055:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575385.731055:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575385.731056:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575385.731057:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575385.731058:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575385.731059:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575385.731060:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575385.731061:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788575385.731062:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575385.731062:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575385.731063:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7800. 00000100:00000001:0.0:1788575385.731064:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575385.731065:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.731066:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575385.731068:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516816384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575385.731078:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575385.731079:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575385.731080:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575385.731082:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575385.731083:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575385.731083:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575385.731084:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575385.731085:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35400. 02000000:00000001:0.0:1788575385.731086:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575385.731087:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575385.731088:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788575385.731089:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575385.731090:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.731091:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575385.731094:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575385.731095:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575385.731096:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575385.731097:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575386.240923:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575386.240929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575386.240932:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575386.240937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575386.240944:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575386.240947:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575386.240950:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575386.240954:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575386.240957:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575386.241017:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575386.241021:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575386.241023:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575386.241025:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575386.241028:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575386.241029:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575386.241032:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575386.241033:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575386.241035:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575386.945013:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575386.945017:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575386.945021:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575387.264917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575387.264923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575387.264926:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575387.264932:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575387.264937:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575387.264941:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575387.264955:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575387.264958:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575387.264963:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575388.288907:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575388.288915:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575388.288918:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575388.288924:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575388.288931:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575388.288938:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575388.288943:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575388.288946:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575388.288950:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575388.288982:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575388.288987:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575388.288990:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575388.288994:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575388.288999:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575388.289003:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575388.289007:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575388.289009:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575388.289012:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575388.993021:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575388.993026:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575388.993029:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575389.312939:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575389.312947:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575389.312950:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575389.312957:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575389.312962:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575389.312989:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575389.312995:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575389.312997:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575389.313003:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575389.568917:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310749421, at 4310749440 00000100:00000010:0.0:1788575389.568932:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575390.017024:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575390.017027:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575390.017030:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575390.336914:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575390.336924:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575390.336930:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575390.336939:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575390.336955:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575390.336963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.336998:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575390.337003:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.337007:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.849200:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575390.849205:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575390.849209:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575390.849218:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5f80 x1875440516816512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.849231:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575390.849234:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575390.849237:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575390.849241:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575390.849246:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575390.849253:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516816512:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575390.849258:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575390.849262:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575390.849266:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575390.849268:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575390.849274:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575390.849279:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575390.849292:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575390.849296:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575390.849304:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575390.849312:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516816512, portal 25 00000100:00000001:0.0:1788575390.849317:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575390.849320:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605328768 : -123400104222848 : ffff8fc4ac9f5f80) 00000100:00000040:0.0:1788575390.849332:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5f80 x1875440516816512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.849347:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575390.849352:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575390.849356:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575390.849362:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516816512, offset 0 00000400:00000200:0.0:1788575390.849372:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575390.849380:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575390.849386:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575390.849391:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575390.849396:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575390.849401:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575390.849406:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575390.849411:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548031:548031:256:4294967295] 192.168.202.128@tcp LPNI seq info [547954:547954:8:4294967295] 00000400:00000200:0.0:1788575390.849420:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575390.849432:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575390.849440:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575390.849477:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575390.849488:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575390.849495:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575390.849516:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575390.849519:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575390.849526:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575390.849534:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575390.849540:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575390.849549:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575390.849554:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575390.849560:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5f80 x1875440516816512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.849572:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575390.849575:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5f80 x1875440516816512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.849586:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.849589:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575390.849597:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575390.849607:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575390.849612:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310754541 00000100:00000010:0.0:1788575390.849622:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575390.849631:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575390.849640:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310754541 00000100:00000010:0.0:1788575390.849648:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50ea0. 00000100:00000001:0.0:1788575390.849654:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575390.849659:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575390.849678:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.849681:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575390.849687:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575390/real 1788575390] req@ffff8fc4ac9f5f80 x1875440516816512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575390.849707:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575390.849711:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575390.849715:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575390.849720:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575390.849724:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575390.849728:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575390.849735:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5f80 x1875440516816512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575390.849748:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5f80 x1875440516816512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.849768:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575390.849771:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.849775:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575390.849778:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575390.849785:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5f80 x1875440516816512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.849792:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575390.849796:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5f80 x1875440516816512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.849804:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575390.849829:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.849830:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575390.849831:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.849833:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575390.849835:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575390.849840:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575390.849846:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575390.849850:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575390.849861:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5f80 x1875440516816512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575390.849884:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516816512:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575390.849896:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.849901:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575390.849908:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5f80 x1875440516816512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.849926:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575390.849930:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575390.849940:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575390.849947:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575390.849951:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575390.849956:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575390.849962:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575390.849966:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788575390.849971:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575390.849975:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575390.849979:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5f80. 00000100:00000001:0.0:1788575390.849984:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575390.849988:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.849992:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.850001:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575390.850004:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.850006:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575390.850007:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575390.850009:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575390.850014:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4700 x1875440516816640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850027:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575390.850029:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575390.850032:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575390.850034:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575390.850036:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575390.850041:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816640:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575390.850045:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575390.850049:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575390.850053:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575390.850055:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575390.850059:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575390.850063:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575390.850070:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575390.850074:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575390.850079:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575390.850083:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516816640, portal 10 00000100:00000001:0.0:1788575390.850092:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575390.850094:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605322496 : -123400104229120 : ffff8fc4ac9f4700) 00000100:00000040:0.0:1788575390.850102:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4700 x1875440516816640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850128:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575390.850134:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575390.850137:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575390.850140:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516816640, offset 0 00000400:00000200:0.0:1788575390.850147:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575390.850152:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575390.850159:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575390.850166:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575390.850171:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575390.850175:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575390.850178:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575390.850182:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548032:548032:256:4294967295] 192.168.202.128@tcp LPNI seq info [547955:547955:8:4294967295] 00000400:00000200:0.0:1788575390.850190:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575390.850200:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575390.850209:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575390.850213:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575390.850220:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575390.850223:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575390.850238:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575390.850241:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575390.850246:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575390.850252:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575390.850256:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575390.850259:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575390.850262:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575390.850266:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4700 x1875440516816640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850292:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575390.850297:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4700 x1875440516816640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850313:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.850319:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575390.850326:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575390.850333:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575390.850342:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575390.850346:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310750739 00000100:00000001:0.0:1788575390.850351:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310754541 00000100:00000001:0.0:1788575390.850353:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310750739 00000100:00000001:0.0:1788575390.850355:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310754541 00000100:00000001:0.0:1788575390.850360:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575390.850367:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.850369:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575390.850376:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575390/real 1788575390] req@ffff8fc4ac9f4700 x1875440516816640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575390.850390:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575390.850394:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575390.850397:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575390.850402:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575390.850406:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575390.850411:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575390.850419:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4700 x1875440516816640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575390.850438:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4700 x1875440516816640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850447:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575390.850454:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.850457:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575390.850458:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575390.850463:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4700 x1875440516816640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850488:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575390.850493:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4700 x1875440516816640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850502:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575390.850506:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.850507:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575390.850509:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.850511:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575390.850514:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575390.850518:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575390.850524:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575390.850528:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575390.850535:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4700 x1875440516816640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575390.850544:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816640:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575390.850551:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575390.850554:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516816768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850558:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575390.850559:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575390.850561:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575390.850562:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575390.850564:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575390.850570:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816768:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575390.850572:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575390.850574:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575390.850575:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575390.850576:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575390.850579:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575390.850580:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575390.850583:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575390.850585:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575390.850587:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575390.850589:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516816768, portal 10 00000100:00000001:0.0:1788575390.850590:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575390.850591:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788575390.850596:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516816768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850600:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575390.850601:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575390.850603:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575390.850604:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516816768, offset 0 00000400:00000200:0.0:1788575390.850607:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575390.850610:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575390.850612:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575390.850615:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575390.850616:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575390.850618:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575390.850620:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575390.850622:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548033:548033:256:4294967295] 192.168.202.128@tcp LPNI seq info [547956:547956:8:4294967295] 00000400:00000200:0.0:1788575390.850625:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575390.850629:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575390.850632:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575390.850634:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575390.850642:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575390.850644:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575390.850651:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575390.850652:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575390.850654:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575390.850657:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575390.850659:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575390.850660:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575390.850662:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575390.850679:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516816768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850686:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575390.850688:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516816768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850693:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.850694:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575390.850697:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575390.850699:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575390.850701:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575390.850703:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310750746 00000100:00000001:0.0:1788575390.850704:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310754541 00000100:00000001:0.0:1788575390.850706:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310750746 00000100:00000001:0.0:1788575390.850707:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310754541 00000100:00000001:0.0:1788575390.850709:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575390.850710:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.850712:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575390.850714:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575390/real 1788575390] req@ffff8fc4ac9f6a00 x1875440516816768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575390.850719:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575390.850724:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575390.850725:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575390.850726:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575390.850727:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575390.850730:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575390.850732:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516816768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575390.850738:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516816768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850741:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575390.850742:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.850743:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575390.850744:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575390.850747:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516816768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850751:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575390.850753:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516816768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850758:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575390.850759:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.850760:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575390.850761:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.850762:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575390.850764:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575390.850766:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575390.850768:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575390.850769:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575390.850771:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516816768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575390.850783:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816768:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575390.850785:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.850787:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575390.850789:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4700 x1875440516816640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850793:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575390.850794:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575390.850796:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575390.850797:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575390.850799:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575390.850800:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575390.850801:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575390.850802:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788575390.850804:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575390.850805:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575390.850806:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4700. 00000100:00000001:0.0:1788575390.850823:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575390.850824:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.850825:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575390.850827:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516816768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850832:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575390.850832:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575390.850833:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575390.850835:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575390.850836:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575390.850837:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575390.850838:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575390.850839:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37c00. 02000000:00000001:0.0:1788575390.850840:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575390.850841:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575390.850842:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788575390.850847:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575390.850847:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.850849:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.850851:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575390.850852:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.850854:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575390.850855:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575390.850856:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575390.850859:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5f80 x1875440516816896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850864:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575390.850865:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575390.850867:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575390.850869:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575390.850871:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575390.850874:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816896:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575390.850878:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575390.850879:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575390.850881:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575390.850882:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575390.850886:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575390.850887:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575390.850892:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575390.850895:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575390.850899:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575390.850902:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516816896, portal 4 00000100:00000001:0.0:1788575390.850904:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575390.850905:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605328768 : -123400104222848 : ffff8fc4ac9f5f80) 00000100:00000040:0.0:1788575390.850911:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5f80 x1875440516816896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.850919:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575390.850921:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575390.850923:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575390.850926:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516816896, offset 0 00000400:00000200:0.0:1788575390.850935:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575390.850939:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575390.850943:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575390.850946:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575390.850949:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575390.850951:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575390.850954:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575390.850957:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548034:548034:256:4294967295] 192.168.202.128@tcp LPNI seq info [547957:547957:8:4294967295] 00000400:00000200:0.0:1788575390.850961:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575390.850966:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575390.850970:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575390.850972:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575390.850975:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575390.850977:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575390.850984:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575390.850985:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575390.850987:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575390.850989:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575390.850992:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575390.850993:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575390.850995:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575390.850997:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5f80 x1875440516816896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851003:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575390.851005:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5f80 x1875440516816896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851009:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.851010:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575390.851012:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575390.851014:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575390.851016:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575390.851022:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310750887 00000100:00000001:0.0:1788575390.851023:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310754541 00000100:00000001:0.0:1788575390.851025:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310750887 00000100:00000001:0.0:1788575390.851026:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310754541 00000100:00000001:0.0:1788575390.851028:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575390.851029:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.851030:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575390.851033:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575390/real 1788575390] req@ffff8fc4ac9f5f80 x1875440516816896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575390.851038:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575390.851039:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575390.851040:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575390.851042:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575390.851043:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575390.851045:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575390.851047:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5f80 x1875440516816896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575390.851052:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5f80 x1875440516816896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851056:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575390.851057:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.851058:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575390.851059:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575390.851061:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5f80 x1875440516816896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851065:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575390.851068:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5f80 x1875440516816896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851075:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575390.851083:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.851084:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575390.851086:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.851087:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575390.851089:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575390.851092:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575390.851096:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575390.851098:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575390.851103:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5f80 x1875440516816896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575390.851111:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516816896:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575390.851115:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575390.851119:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5180 x1875440516817024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851126:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575390.851128:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575390.851130:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575390.851132:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575390.851134:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575390.851136:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516817024:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575390.851139:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575390.851141:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575390.851143:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575390.851145:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575390.851148:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575390.851150:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575390.851153:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575390.851156:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575390.851159:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575390.851162:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516817024, portal 4 00000100:00000001:0.0:1788575390.851169:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575390.851171:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605325184 : -123400104226432 : ffff8fc4ac9f5180) 00000100:00000040:0.0:1788575390.851177:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5180 x1875440516817024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851182:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575390.851184:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575390.851186:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575390.851187:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516817024, offset 0 00000400:00000200:0.0:1788575390.851190:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575390.851193:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575390.851195:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575390.851197:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575390.851199:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575390.851200:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575390.851202:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575390.851204:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548035:548035:256:4294967295] 192.168.202.128@tcp LPNI seq info [547958:547958:8:4294967295] 00000400:00000200:0.0:1788575390.851207:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575390.851211:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575390.851214:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575390.851216:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575390.851218:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575390.851220:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575390.851231:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575390.851232:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575390.851234:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575390.851236:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575390.851239:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575390.851240:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575390.851242:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575390.851244:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5180 x1875440516817024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851256:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575390.851258:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5180 x1875440516817024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851263:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.851264:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575390.851266:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575390.851268:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575390.851270:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575390.851272:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310750889 00000100:00000001:0.0:1788575390.851274:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310754541 00000100:00000001:0.0:1788575390.851275:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310750889 00000100:00000001:0.0:1788575390.851277:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310754541 00000100:00000001:0.0:1788575390.851278:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575390.851279:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.851280:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575390.851282:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575390/real 1788575390] req@ffff8fc4ac9f5180 x1875440516817024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575390.851287:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575390.851288:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575390.851289:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575390.851291:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575390.851292:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575390.851293:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575390.851295:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516817024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575390.851304:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516817024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851311:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575390.851318:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.851320:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575390.851322:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575390.851326:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516817024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851333:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575390.851337:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516817024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851345:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575390.851347:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.851348:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575390.851349:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.851350:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575390.851351:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575390.851353:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575390.851355:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575390.851357:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575390.851359:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516817024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575390.851364:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516817024:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575390.851366:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.851368:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575390.851370:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5f80 x1875440516816896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851374:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575390.851375:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575390.851377:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575390.851379:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575390.851380:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575390.851384:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575390.851386:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575390.851388:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788575390.851391:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575390.851392:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575390.851394:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5f80. 00000100:00000001:0.0:1788575390.851396:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575390.851398:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.851400:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575390.851403:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516817024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575390.851410:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575390.851412:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575390.851413:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575390.851416:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575390.851417:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575390.851418:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575390.851421:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575390.851422:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35400. 02000000:00000001:0.0:1788575390.851424:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575390.851426:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575390.851427:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788575390.851429:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575390.851431:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.851433:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575390.851437:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575390.851438:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575390.851440:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575390.851441:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575391.360922:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575391.360931:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575391.360936:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575391.360954:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575391.360965:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575391.360970:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575391.360979:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575391.360981:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575391.361000:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575392.065047:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575392.065052:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575392.065055:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575392.384916:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575392.384925:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575392.384930:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575392.384939:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575392.384955:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575392.384963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575392.384969:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575392.384973:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575392.384976:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575393.408912:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575393.408920:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575393.408923:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575393.408931:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575393.408939:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575393.408945:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575393.408960:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575393.408964:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575393.408968:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575394.113006:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575394.113010:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575394.113014:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575394.688884:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310754541, at 4310754560 00000100:00000010:0.0:1788575394.688895:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575395.137012:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575395.137015:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575395.137018:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575395.456937:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575395.456942:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575395.456944:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575395.456947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575395.456952:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575395.456957:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.456979:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575395.456999:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.457003:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.457032:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575395.457036:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575395.457037:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575395.457039:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575395.457042:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575395.457044:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.457046:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575395.457047:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.457049:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.969159:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575395.969164:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575395.969169:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575395.969178:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516817152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.969188:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575395.969191:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575395.969196:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575395.969200:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575395.969204:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575395.969209:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516817152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575395.969213:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575395.969218:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575395.969221:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575395.969225:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575395.969230:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575395.969234:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575395.969242:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575395.969246:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575395.969254:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575395.969261:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516817152, portal 25 00000100:00000001:0.0:1788575395.969265:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575395.969267:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575395.969274:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516817152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.969301:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575395.969306:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575395.969312:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575395.969316:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516817152, offset 0 00000400:00000200:0.0:1788575395.969325:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575395.969335:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575395.969343:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575395.969349:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575395.969356:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575395.969360:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575395.969366:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575395.969372:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548036:548036:256:4294967295] 192.168.202.128@tcp LPNI seq info [547959:547959:8:4294967295] 00000400:00000200:0.0:1788575395.969380:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575395.969399:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575395.969414:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575395.969424:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575395.969435:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575395.969439:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575395.969460:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575395.969464:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575395.969496:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575395.969506:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575395.969511:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575395.969514:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575395.969529:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575395.969538:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516817152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.969556:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575395.969564:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516817152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.969593:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.969627:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575395.969639:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575395.969648:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575395.969654:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310759661 00000100:00000010:0.0:1788575395.969665:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575395.969675:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575395.969684:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310759661 00000100:00000010:0.0:1788575395.969691:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50480. 00000100:00000001:0.0:1788575395.969697:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575395.969704:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575395.969709:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.969714:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575395.969725:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575395/real 1788575395] req@ffff8fc4901a7800 x1875440516817152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575395.969746:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575395.969749:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575395.969752:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575395.969755:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575395.969758:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575395.969762:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575395.969768:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516817152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575395.969793:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516817152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.969835:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575395.969837:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.969840:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575395.969843:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575395.969853:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516817152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.969874:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575395.969884:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516817152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.969902:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575395.969910:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.969914:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575395.969917:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.969921:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575395.969925:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575395.969931:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575395.969940:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575395.969946:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575395.969963:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516817152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575395.969992:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516817152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575395.970003:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.970009:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575395.970016:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516817152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970028:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575395.970033:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575395.970039:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575395.970046:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575395.970051:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575395.970056:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575395.970063:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575395.970068:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575395.970076:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575395.970086:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575395.970091:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575395.970099:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575395.970103:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.970108:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.970120:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575395.970125:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.970131:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575395.970135:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575395.970139:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575395.970148:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516817280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970168:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575395.970172:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575395.970179:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575395.970183:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575395.970194:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575395.970204:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516817280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575395.970212:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575395.970217:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575395.970222:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575395.970228:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575395.970236:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575395.970242:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575395.970251:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575395.970255:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575395.970260:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575395.970264:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516817280, portal 10 00000100:00000001:0.0:1788575395.970266:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575395.970268:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575395.970278:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516817280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970286:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575395.970291:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575395.970300:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575395.970303:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516817280, offset 0 00000400:00000200:0.0:1788575395.970309:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575395.970314:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575395.970319:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575395.970324:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575395.970327:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575395.970330:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575395.970333:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575395.970336:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548037:548037:256:4294967295] 192.168.202.128@tcp LPNI seq info [547960:547960:8:4294967295] 00000400:00000200:0.0:1788575395.970340:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575395.970348:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575395.970353:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575395.970356:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575395.970361:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575395.970363:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575395.970375:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575395.970377:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575395.970380:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575395.970384:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575395.970387:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575395.970390:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575395.970393:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575395.970397:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516817280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970404:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575395.970408:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516817280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970414:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.970416:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575395.970427:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575395.970431:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575395.970434:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575395.970437:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310755859 00000100:00000001:0.0:1788575395.970439:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310759661 00000100:00000001:0.0:1788575395.970442:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310755859 00000100:00000001:0.0:1788575395.970444:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310759661 00000100:00000001:0.0:1788575395.970446:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575395.970448:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.970450:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575395.970454:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575395/real 1788575395] req@ffff8fc4901a5c00 x1875440516817280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575395.970463:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575395.970465:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575395.970466:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575395.970469:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575395.970470:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575395.970473:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575395.970476:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516817280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575395.970484:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516817280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970490:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575395.970491:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.970493:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575395.970495:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575395.970498:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516817280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970504:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575395.970508:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516817280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970520:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575395.970522:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.970524:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575395.970525:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.970527:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575395.970529:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575395.970531:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575395.970535:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575395.970538:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575395.970542:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516817280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575395.970549:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516817280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575395.970553:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575395.970557:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516817408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970563:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575395.970565:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575395.970567:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575395.970569:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575395.970571:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575395.970574:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516817408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575395.970577:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575395.970579:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575395.970581:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575395.970582:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575395.970586:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575395.970588:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575395.970612:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575395.970615:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575395.970619:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575395.970622:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516817408, portal 10 00000100:00000001:0.0:1788575395.970624:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575395.970625:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575395.970632:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516817408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970638:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575395.970640:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575395.970643:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575395.970645:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516817408, offset 0 00000400:00000200:0.0:1788575395.970649:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575395.970653:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575395.970657:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575395.970660:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575395.970663:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575395.970665:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575395.970667:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575395.970670:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548038:548038:256:4294967295] 192.168.202.128@tcp LPNI seq info [547961:547961:8:4294967295] 00000400:00000200:0.0:1788575395.970674:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575395.970681:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575395.970685:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575395.970688:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575395.970692:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575395.970694:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575395.970705:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575395.970707:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575395.970710:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575395.970714:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575395.970717:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575395.970725:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575395.970729:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575395.970733:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516817408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970742:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575395.970745:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516817408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970752:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.970753:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575395.970757:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575395.970761:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575395.970765:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575395.970767:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310755866 00000100:00000001:0.0:1788575395.970770:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310759661 00000100:00000001:0.0:1788575395.970773:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310755866 00000100:00000001:0.0:1788575395.970775:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310759661 00000100:00000001:0.0:1788575395.970778:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575395.970780:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.970782:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575395.970786:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575395/real 1788575395] req@ffff8fc4901a5180 x1875440516817408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575395.970794:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575395.970795:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575395.970796:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575395.970798:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575395.970799:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575395.970801:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575395.970803:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516817408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575395.970833:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516817408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970847:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575395.970849:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.970851:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575395.970853:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575395.970858:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516817408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970866:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575395.970871:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516817408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970878:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575395.970881:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.970882:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575395.970884:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.970885:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575395.970888:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575395.970891:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575395.970895:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575395.970898:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575395.970903:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516817408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575395.970912:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516817408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575395.970916:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575395.970920:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516817536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970928:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575395.970929:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575395.970938:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575395.970940:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575395.970942:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575395.970946:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516817536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575395.970950:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575395.970952:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575395.970954:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575395.970956:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575395.970960:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575395.970962:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575395.970967:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575395.970970:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575395.970974:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575395.970977:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516817536, portal 4 00000100:00000001:0.0:1788575395.970980:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575395.970982:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575395.970988:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516817536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.970996:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575395.970999:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575395.971002:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575395.971004:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516817536, offset 0 00000400:00000200:0.0:1788575395.971010:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575395.971015:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575395.971020:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575395.971023:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575395.971026:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575395.971029:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575395.971031:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575395.971034:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548039:548039:256:4294967295] 192.168.202.128@tcp LPNI seq info [547962:547962:8:4294967295] 00000400:00000200:0.0:1788575395.971038:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575395.971044:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575395.971056:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575395.971060:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575395.971064:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575395.971067:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575395.971083:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575395.971084:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575395.971087:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575395.971089:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575395.971092:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575395.971093:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575395.971095:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575395.971098:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516817536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971104:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575395.971106:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516817536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971110:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.971112:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575395.971114:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575395.971117:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575395.971119:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575395.971121:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310756007 00000100:00000001:0.0:1788575395.971122:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310759661 00000100:00000001:0.0:1788575395.971124:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310756007 00000100:00000001:0.0:1788575395.971125:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310759661 00000100:00000001:0.0:1788575395.971127:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575395.971128:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.971129:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575395.971132:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575395/real 1788575395] req@ffff8fc4901a7100 x1875440516817536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575395.971142:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575395.971143:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575395.971144:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575395.971146:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575395.971147:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575395.971148:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575395.971151:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516817536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575395.971156:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516817536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971161:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575395.971162:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971163:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575395.971164:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575395.971166:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516817536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971170:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575395.971172:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516817536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971178:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575395.971181:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971182:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575395.971183:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971185:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575395.971187:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575395.971189:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575395.971193:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575395.971195:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575395.971205:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516817536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575395.971213:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516817536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575395.971218:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971220:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575395.971224:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516817280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971232:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575395.971234:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575395.971235:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575395.971238:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575395.971240:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575395.971241:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575395.971242:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575395.971244:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575395.971246:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575395.971247:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575395.971248:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575395.971253:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575395.971254:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971255:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575395.971258:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516817408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971262:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575395.971263:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575395.971264:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575395.971265:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575395.971266:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575395.971266:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575395.971268:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575395.971268:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575395.971270:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575395.971275:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575395.971277:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575395.971278:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575395.971279:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971280:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575395.971283:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516817536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971287:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575395.971287:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575395.971288:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575395.971290:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575395.971290:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575395.971291:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575395.971292:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575395.971293:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575395.971295:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575395.971296:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575395.971297:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575395.971299:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575395.971300:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971301:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971305:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575395.971307:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.971309:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575395.971310:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575395.971312:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575395.971315:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516817664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971322:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575395.971324:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575395.971326:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575395.971328:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575395.971331:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575395.971334:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516817664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575395.971337:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575395.971344:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575395.971346:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575395.971348:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575395.971350:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575395.971352:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575395.971356:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575395.971359:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575395.971362:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575395.971365:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516817664, portal 4 00000100:00000001:0.0:1788575395.971368:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575395.971369:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575395.971376:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516817664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971383:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575395.971385:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575395.971388:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575395.971391:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516817664, offset 0 00000400:00000200:0.0:1788575395.971398:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575395.971401:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575395.971404:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575395.971406:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575395.971408:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575395.971410:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575395.971412:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575395.971414:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548040:548040:256:4294967295] 192.168.202.128@tcp LPNI seq info [547963:547963:8:4294967295] 00000400:00000200:0.0:1788575395.971417:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575395.971422:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575395.971425:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575395.971428:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575395.971430:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575395.971432:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575395.971439:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575395.971445:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575395.971447:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575395.971450:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575395.971452:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575395.971453:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575395.971455:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575395.971458:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516817664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971463:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575395.971465:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516817664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971470:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.971471:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575395.971473:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575395.971475:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575395.971477:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575395.971478:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310756009 00000100:00000001:0.0:1788575395.971480:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310759661 00000100:00000001:0.0:1788575395.971482:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310756009 00000100:00000001:0.0:1788575395.971484:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310759661 00000100:00000001:0.0:1788575395.971485:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575395.971486:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.971487:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575395.971490:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575395/real 1788575395] req@ffff8fc4901a4a80 x1875440516817664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575395.971495:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575395.971496:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575395.971497:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575395.971498:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575395.971499:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575395.971504:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575395.971507:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516817664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575395.971512:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516817664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971516:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575395.971517:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971518:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575395.971519:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575395.971521:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516817664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971526:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575395.971528:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516817664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971532:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575395.971534:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971535:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575395.971536:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971537:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575395.971538:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575395.971540:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575395.971542:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575395.971543:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575395.971545:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516817664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575395.971550:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516817664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575395.971552:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971553:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575395.971559:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516817664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575395.971563:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575395.971564:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575395.971566:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575395.971567:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575395.971568:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575395.971569:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575395.971570:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575395.971571:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575395.971573:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575395.971574:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575395.971575:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575395.971578:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575395.971579:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971580:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575395.971583:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575395.971584:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575395.971585:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575395.971586:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575396.480922:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575396.480932:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575396.480937:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575396.480945:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575396.480953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575396.480959:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575396.480966:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575396.480973:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575396.480980:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575396.481018:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575396.481023:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575396.481025:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575396.481027:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575396.481030:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575396.481034:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575396.481037:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575396.481039:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575396.481049:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575397.185058:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575397.185063:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575397.185066:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575397.504916:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575397.504924:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575397.504929:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575397.504938:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575397.504947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575397.504953:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575397.504968:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575397.504972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575397.504977:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575397.505037:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575397.505041:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575397.505043:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575397.505047:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575397.505050:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575397.505052:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575397.505055:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575397.505056:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575397.505059:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575398.528929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575398.528938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575398.528944:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575398.528953:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575398.528958:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575398.528965:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575398.528972:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575398.528976:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575398.528980:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575398.529063:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575398.529066:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575398.529068:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575398.529071:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575398.529074:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575398.529077:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575398.529608:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575398.529631:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575398.529637:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575399.232984:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575399.232988:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575399.232991:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575399.552910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575399.552915:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575399.552917:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575399.552923:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575399.552931:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575399.552938:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575399.552944:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575399.552947:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575399.552951:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575399.552977:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575399.552981:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575399.552983:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575399.552985:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575399.552988:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575399.552991:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575399.552994:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575399.552996:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575399.552998:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575399.808912:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310759661, at 4310759680 00000100:00000010:0.0:1788575399.808928:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575400.256969:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575400.256972:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575400.256974:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575400.576965:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575400.576974:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575400.576982:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575400.576992:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575400.577003:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575400.577015:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575400.577020:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575400.577027:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575400.577031:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575400.577074:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575400.577079:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575400.577082:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575400.577085:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575400.577090:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575400.577093:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575400.577096:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575400.577099:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575400.577101:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.089214:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575401.089225:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575401.089230:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575401.089247:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516817792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.089275:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575401.089279:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575401.089287:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575401.089293:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575401.089304:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575401.089320:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516817792:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575401.089333:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575401.089341:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575401.089346:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575401.089351:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575401.089363:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575401.089369:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575401.089399:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575401.089409:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575401.089425:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575401.089441:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516817792, portal 25 00000100:00000001:0.0:1788575401.089450:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575401.089455:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788575401.089473:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516817792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.089575:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575401.089589:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575401.089598:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575401.089610:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516817792, offset 0 00000400:00000200:0.0:1788575401.089634:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575401.089650:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575401.089670:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575401.089685:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575401.089695:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575401.089705:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575401.089712:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575401.089725:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548041:548041:256:4294967295] 192.168.202.128@tcp LPNI seq info [547964:547964:8:4294967295] 00000400:00000200:0.0:1788575401.089741:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575401.089774:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575401.089792:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575401.089801:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575401.089879:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575401.089888:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575401.089928:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575401.089936:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575401.089949:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575401.089967:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575401.089983:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575401.089996:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575401.090011:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575401.090025:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516817792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090056:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575401.090065:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516817792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090122:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.090132:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575401.090143:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575401.090157:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575401.090163:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310764782 00000100:00000010:0.0:1788575401.090175:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575401.090185:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575401.090199:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310764782 00000100:00000010:0.0:1788575401.090205:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b502a0. 00000100:00000001:0.0:1788575401.090209:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575401.090214:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575401.090217:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.090220:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575401.090227:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575401/real 1788575401] req@ffff8fc4ac822d80 x1875440516817792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575401.090238:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575401.090240:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575401.090242:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575401.090245:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575401.090247:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575401.090250:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575401.090254:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516817792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575401.090263:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516817792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090271:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575401.090273:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.090275:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575401.090287:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575401.090295:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516817792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090309:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575401.090315:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516817792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090323:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575401.090326:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.090328:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575401.090329:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.090332:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575401.090334:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575401.090338:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575401.090342:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575401.090345:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575401.090349:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516817792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575401.090357:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516817792:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575401.090362:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.090364:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575401.090367:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516817792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090374:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575401.090377:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575401.090380:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575401.090383:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575401.090386:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575401.090388:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575401.090390:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575401.090393:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788575401.090397:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575401.090403:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575401.090406:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788575401.090410:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575401.090411:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.090414:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.090422:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575401.090424:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.090426:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575401.090429:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575401.090430:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575401.090435:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516817920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090442:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575401.090443:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575401.090446:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575401.090448:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575401.090450:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575401.090453:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516817920:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575401.090457:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575401.090459:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575401.090461:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575401.090463:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575401.090466:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575401.090468:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575401.090472:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575401.090475:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575401.090478:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575401.090481:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516817920, portal 10 00000100:00000001:0.0:1788575401.090483:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575401.090485:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788575401.090492:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516817920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090499:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575401.090502:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575401.090508:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575401.090510:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516817920, offset 0 00000400:00000200:0.0:1788575401.090514:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575401.090518:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575401.090522:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575401.090525:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575401.090528:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575401.090530:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575401.090533:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575401.090536:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548042:548042:256:4294967295] 192.168.202.128@tcp LPNI seq info [547965:547965:8:4294967295] 00000400:00000200:0.0:1788575401.090540:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575401.090547:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575401.090564:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575401.090567:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575401.090571:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575401.090574:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575401.090585:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575401.090586:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575401.090589:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575401.090593:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575401.090596:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575401.090598:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575401.090601:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575401.090605:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516817920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090613:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575401.090616:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516817920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090624:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.090625:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575401.090634:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575401.090638:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575401.090641:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575401.090643:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310760979 00000100:00000001:0.0:1788575401.090646:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310764782 00000100:00000001:0.0:1788575401.090648:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310760979 00000100:00000001:0.0:1788575401.090650:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310764782 00000100:00000001:0.0:1788575401.090652:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575401.090654:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.090656:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575401.090660:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575401/real 1788575401] req@ffff8fc4ac821f80 x1875440516817920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575401.090668:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575401.090669:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575401.090671:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575401.090674:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575401.090675:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575401.090678:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575401.090681:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516817920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575401.090689:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516817920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090696:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575401.090698:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.090699:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575401.090701:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575401.090704:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516817920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090711:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575401.090715:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516817920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090726:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575401.090729:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.090730:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575401.090731:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.090733:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575401.090736:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575401.090738:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575401.090742:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575401.090744:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575401.090748:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516817920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575401.090756:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516817920:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575401.090761:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575401.090764:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516818048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090771:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575401.090772:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575401.090774:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575401.090776:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575401.090778:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575401.090781:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818048:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575401.090785:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575401.090786:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575401.090788:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575401.090789:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575401.090793:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575401.090795:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575401.090803:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575401.090806:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575401.090831:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575401.090835:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516818048, portal 10 00000100:00000001:0.0:1788575401.090836:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575401.090838:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788575401.090845:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516818048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090851:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575401.090853:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575401.090856:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575401.090858:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516818048, offset 0 00000400:00000200:0.0:1788575401.090861:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575401.090865:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575401.090869:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575401.090871:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575401.090874:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575401.090876:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575401.090878:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575401.090881:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548043:548043:256:4294967295] 192.168.202.128@tcp LPNI seq info [547966:547966:8:4294967295] 00000400:00000200:0.0:1788575401.090885:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575401.090891:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575401.090897:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575401.090900:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575401.090904:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575401.090906:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575401.090916:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575401.090919:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575401.090921:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575401.090924:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575401.090928:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575401.090935:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575401.090938:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575401.090941:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516818048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090949:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575401.090952:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516818048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.090958:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.090960:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575401.090963:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575401.090967:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575401.090970:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575401.090973:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310760987 00000100:00000001:0.0:1788575401.090975:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310764782 00000100:00000001:0.0:1788575401.090977:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310760987 00000100:00000001:0.0:1788575401.090979:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310764782 00000100:00000001:0.0:1788575401.090981:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575401.090983:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.090985:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575401.090989:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575401/real 1788575401] req@ffff8fc4ac823480 x1875440516818048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575401.090997:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575401.090998:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575401.091000:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575401.091002:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575401.091004:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575401.091006:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575401.091009:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516818048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575401.091017:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516818048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091029:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575401.091031:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091033:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575401.091034:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575401.091038:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516818048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091045:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575401.091049:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516818048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091056:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575401.091058:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091059:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575401.091061:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091063:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575401.091066:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575401.091069:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575401.091072:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575401.091074:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575401.091078:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516818048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575401.091085:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818048:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575401.091089:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575401.091092:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516818176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091100:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575401.091101:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575401.091109:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575401.091111:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575401.091113:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575401.091116:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818176:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575401.091120:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575401.091122:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575401.091123:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575401.091125:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575401.091128:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575401.091129:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575401.091133:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575401.091136:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575401.091139:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575401.091142:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516818176, portal 4 00000100:00000001:0.0:1788575401.091144:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575401.091145:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788575401.091150:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516818176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091157:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575401.091159:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575401.091161:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575401.091163:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516818176, offset 0 00000400:00000200:0.0:1788575401.091167:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575401.091171:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575401.091175:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575401.091178:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575401.091180:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575401.091182:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575401.091184:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575401.091188:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548044:548044:256:4294967295] 192.168.202.128@tcp LPNI seq info [547967:547967:8:4294967295] 00000400:00000200:0.0:1788575401.091192:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575401.091198:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575401.091207:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575401.091210:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575401.091214:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575401.091216:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575401.091226:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575401.091228:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575401.091231:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575401.091234:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575401.091237:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575401.091239:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575401.091242:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575401.091246:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516818176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091253:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575401.091257:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516818176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091263:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.091264:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575401.091267:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575401.091271:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575401.091274:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575401.091276:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310761127 00000100:00000001:0.0:1788575401.091279:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310764782 00000100:00000001:0.0:1788575401.091281:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310761127 00000100:00000001:0.0:1788575401.091283:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310764782 00000100:00000001:0.0:1788575401.091285:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575401.091287:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.091288:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575401.091291:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575401/real 1788575401] req@ffff8fc4ac820700 x1875440516818176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575401.091304:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575401.091305:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575401.091307:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575401.091310:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575401.091311:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575401.091313:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575401.091317:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516818176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575401.091326:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516818176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091332:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575401.091333:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091335:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575401.091337:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575401.091341:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516818176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091347:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575401.091350:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516818176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091357:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575401.091359:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091360:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575401.091362:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091364:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575401.091365:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575401.091368:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575401.091371:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575401.091373:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575401.091383:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516818176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575401.091391:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818176:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575401.091395:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575401.091398:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516818304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091405:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575401.091407:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575401.091409:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575401.091411:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575401.091413:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575401.091415:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818304:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575401.091419:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575401.091420:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575401.091422:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575401.091424:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575401.091427:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575401.091429:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575401.091432:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575401.091435:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575401.091438:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575401.091441:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516818304, portal 4 00000100:00000001:0.0:1788575401.091443:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575401.091445:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788575401.091450:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516818304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091458:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575401.091460:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575401.091462:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575401.091465:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516818304, offset 0 00000400:00000200:0.0:1788575401.091473:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575401.091477:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575401.091480:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575401.091483:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575401.091485:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575401.091487:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575401.091489:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575401.091492:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548045:548045:256:4294967295] 192.168.202.128@tcp LPNI seq info [547968:547968:8:4294967295] 00000400:00000200:0.0:1788575401.091496:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575401.091502:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575401.091506:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575401.091509:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575401.091513:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575401.091516:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575401.091526:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575401.091528:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575401.091531:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575401.091534:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575401.091537:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575401.091539:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575401.091541:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575401.091545:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516818304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091565:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575401.091568:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516818304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091575:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.091576:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575401.091579:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575401.091583:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575401.091586:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575401.091595:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310761129 00000100:00000001:0.0:1788575401.091598:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310764782 00000100:00000001:0.0:1788575401.091601:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310761129 00000100:00000001:0.0:1788575401.091603:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310764782 00000100:00000001:0.0:1788575401.091605:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575401.091608:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.091609:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575401.091614:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575401/real 1788575401] req@ffff8fc4ac822300 x1875440516818304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575401.091621:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575401.091622:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575401.091624:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575401.091627:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575401.091628:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575401.091631:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575401.091634:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516818304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575401.091642:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516818304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091649:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575401.091650:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091651:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575401.091653:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575401.091656:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516818304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091662:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575401.091666:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516818304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091673:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575401.091681:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091682:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575401.091684:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091685:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575401.091688:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575401.091690:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575401.091694:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575401.091696:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575401.091700:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516818304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575401.091707:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818304:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575401.091711:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091713:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575401.091716:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516817920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091722:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575401.091724:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575401.091726:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575401.091728:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575401.091729:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575401.091731:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575401.091733:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575401.091735:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788575401.091737:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575401.091738:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575401.091740:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788575401.091742:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575401.091744:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091745:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575401.091749:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516818048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091765:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575401.091767:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575401.091768:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575401.091770:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575401.091772:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575401.091773:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575401.091775:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575401.091777:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788575401.091780:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575401.091781:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575401.091782:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788575401.091785:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575401.091786:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091788:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575401.091791:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516818176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091797:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575401.091798:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575401.091799:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575401.091802:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575401.091804:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575401.091805:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575401.091807:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575401.091823:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788575401.091826:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575401.091828:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575401.091829:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788575401.091832:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575401.091833:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091835:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575401.091838:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516818304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575401.091845:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575401.091846:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575401.091851:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575401.091853:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575401.091855:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575401.091856:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575401.091858:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575401.091859:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788575401.091861:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575401.091862:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575401.091864:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788575401.091866:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575401.091867:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091869:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.091872:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575401.091874:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.091875:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575401.091877:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575401.600933:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575401.600943:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575401.600948:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575401.600957:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575401.600966:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575401.600972:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.601005:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575401.601009:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575401.601013:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575402.305017:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575402.305020:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575402.305023:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575402.625075:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575402.625079:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575402.625080:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575402.625082:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575402.625084:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575402.625087:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575402.625234:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575402.625236:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575402.625239:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575403.648936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575403.648963:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575403.648965:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575403.648970:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575403.648974:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575403.648976:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575403.648977:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575403.648979:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575403.648981:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575403.649251:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575403.649254:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575403.649256:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575403.649258:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575403.649259:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575403.649261:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575403.649262:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575403.649263:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575403.649265:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575404.352988:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575404.352991:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575404.352994:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575404.673000:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575404.673006:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575404.673011:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575404.673019:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575404.673029:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575404.673034:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575404.673060:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575404.673066:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575404.673070:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575404.928918:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310764782, at 4310764800 00000100:00000010:0.0:1788575404.928932:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575405.376988:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575405.376992:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575405.376995:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575406.209170:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575406.209176:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575406.209180:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575406.209196:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516818432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.209213:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575406.209216:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575406.209223:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575406.209226:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575406.209240:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575406.209247:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516818432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575406.209254:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575406.209259:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575406.209264:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575406.209270:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575406.209276:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575406.209279:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575406.209285:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575406.209291:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575406.209300:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575406.209309:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516818432, portal 25 00000100:00000001:0.0:1788575406.209314:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575406.209318:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788575406.209329:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516818432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.209346:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575406.209353:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575406.209358:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575406.209364:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516818432, offset 0 00000400:00000200:0.0:1788575406.209374:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575406.209394:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575406.209405:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575406.209412:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575406.209421:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575406.209427:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575406.209434:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575406.209450:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548046:548046:256:4294967295] 192.168.202.128@tcp LPNI seq info [547969:547969:8:4294967295] 00000400:00000200:0.0:1788575406.209460:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575406.209475:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575406.209487:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575406.209495:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575406.209539:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575406.209545:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575406.209570:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575406.209575:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575406.209590:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575406.209597:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575406.209606:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575406.209611:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575406.209619:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575406.209628:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516818432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.209653:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575406.209663:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516818432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.209678:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.209683:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575406.209692:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575406.209700:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575406.209706:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310769901 00000100:00000010:0.0:1788575406.209715:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575406.209724:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575406.209731:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310769901 00000100:00000010:0.0:1788575406.209738:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50540. 00000100:00000001:0.0:1788575406.209746:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575406.209752:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575406.209762:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.209767:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575406.209776:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575406/real 1788575406] req@ffff8fc4ac821c00 x1875440516818432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575406.209795:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575406.209799:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575406.209803:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575406.209831:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575406.209834:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575406.209839:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575406.209847:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516818432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575406.209864:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516818432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.209879:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575406.209883:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.209888:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575406.209891:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575406.209898:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516818432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.209914:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575406.209921:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516818432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.209932:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575406.209937:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.209941:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575406.209943:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.209948:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575406.209951:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575406.209955:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575406.209963:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575406.209974:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575406.209983:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516818432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575406.209995:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516818432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575406.210000:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.210002:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575406.210005:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516818432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210012:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575406.210014:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575406.210016:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575406.210020:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575406.210022:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575406.210023:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575406.210025:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575406.210027:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788575406.210030:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575406.210033:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575406.210034:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788575406.210037:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575406.210038:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.210040:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.210046:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575406.210048:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.210050:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575406.210051:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575406.210052:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575406.210055:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516818560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210061:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575406.210062:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575406.210064:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575406.210065:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575406.210072:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575406.210075:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575406.210078:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575406.210079:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575406.210081:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575406.210082:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575406.210084:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575406.210086:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575406.210089:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575406.210092:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575406.210095:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575406.210098:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516818560, portal 10 00000100:00000001:0.0:1788575406.210100:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575406.210101:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788575406.210107:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516818560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210113:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575406.210115:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575406.210117:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575406.210119:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516818560, offset 0 00000400:00000200:0.0:1788575406.210122:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575406.210126:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575406.210130:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575406.210133:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575406.210136:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575406.210138:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575406.210141:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575406.210143:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548047:548047:256:4294967295] 192.168.202.128@tcp LPNI seq info [547970:547970:8:4294967295] 00000400:00000200:0.0:1788575406.210147:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575406.210153:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575406.210158:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575406.210168:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575406.210172:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575406.210175:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575406.210189:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575406.210191:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575406.210194:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575406.210198:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575406.210201:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575406.210203:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575406.210205:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575406.210209:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516818560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210216:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575406.210219:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516818560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210225:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.210226:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575406.210230:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575406.210233:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575406.210236:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575406.210238:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310766099 00000100:00000001:0.0:1788575406.210241:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310769901 00000100:00000001:0.0:1788575406.210243:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310766099 00000100:00000001:0.0:1788575406.210245:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310769901 00000100:00000001:0.0:1788575406.210247:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575406.210249:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.210251:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575406.210255:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575406/real 1788575406] req@ffff8fc4ac820a80 x1875440516818560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575406.210274:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575406.210275:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575406.210277:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575406.210279:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575406.210281:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575406.210283:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575406.210286:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516818560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575406.210294:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516818560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210300:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575406.210301:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.210303:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575406.210305:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575406.210308:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516818560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210314:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575406.210318:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516818560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210324:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575406.210327:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.210328:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575406.210329:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.210331:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575406.210333:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575406.210335:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575406.210338:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575406.210341:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575406.210350:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516818560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575406.210358:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575406.210361:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575406.210364:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516818688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210370:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575406.210371:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575406.210373:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575406.210375:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575406.210377:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575406.210379:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575406.210382:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575406.210384:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575406.210386:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575406.210387:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575406.210390:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575406.210392:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575406.210396:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575406.210398:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575406.210401:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575406.210404:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516818688, portal 10 00000100:00000001:0.0:1788575406.210406:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575406.210407:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788575406.210412:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516818688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210418:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575406.210420:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575406.210422:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575406.210424:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516818688, offset 0 00000400:00000200:0.0:1788575406.210433:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575406.210437:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575406.210440:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575406.210442:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575406.210444:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575406.210446:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575406.210448:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575406.210451:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548048:548048:256:4294967295] 192.168.202.128@tcp LPNI seq info [547971:547971:8:4294967295] 00000400:00000200:0.0:1788575406.210455:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575406.210461:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575406.210465:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575406.210468:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575406.210472:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575406.210475:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575406.210484:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575406.210486:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575406.210489:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575406.210492:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575406.210494:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575406.210496:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575406.210498:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575406.210515:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516818688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210523:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575406.210526:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516818688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210532:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.210533:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575406.210536:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575406.210540:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575406.210542:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575406.210549:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310766106 00000100:00000001:0.0:1788575406.210552:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310769901 00000100:00000001:0.0:1788575406.210554:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310766106 00000100:00000001:0.0:1788575406.210556:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310769901 00000100:00000001:0.0:1788575406.210558:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575406.210560:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.210561:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575406.210565:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575406/real 1788575406] req@ffff8fc4ac823800 x1875440516818688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575406.210573:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575406.210574:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575406.210576:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575406.210578:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575406.210580:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575406.210582:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575406.210585:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516818688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575406.210593:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516818688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210599:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575406.210600:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.210602:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575406.210603:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575406.210606:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516818688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210612:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575406.210616:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516818688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210622:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575406.210629:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.210630:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575406.210631:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.210633:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575406.210635:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575406.210638:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575406.210641:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575406.210644:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575406.210647:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516818688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575406.210654:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575406.210657:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575406.210661:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516818816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210667:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575406.210669:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575406.210671:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575406.210673:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575406.210675:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575406.210677:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575406.210680:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575406.210682:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575406.210683:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575406.210685:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575406.210688:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575406.210690:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575406.210693:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575406.210696:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575406.210699:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575406.210702:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516818816, portal 4 00000100:00000001:0.0:1788575406.210709:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575406.210711:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788575406.210717:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516818816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210723:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575406.210726:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575406.210728:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575406.210731:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516818816, offset 0 00000400:00000200:0.0:1788575406.210735:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575406.210738:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575406.210742:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575406.210744:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575406.210747:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575406.210749:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575406.210752:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575406.210755:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548049:548049:256:4294967295] 192.168.202.128@tcp LPNI seq info [547972:547972:8:4294967295] 00000400:00000200:0.0:1788575406.210758:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575406.210764:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575406.210768:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575406.210771:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575406.210774:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575406.210777:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575406.210787:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575406.210789:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575406.210791:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575406.210794:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575406.210797:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575406.210799:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575406.210801:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575406.210804:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516818816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210839:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575406.210842:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516818816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210849:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.210850:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575406.210853:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575406.210856:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575406.210859:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575406.210861:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310766247 00000100:00000001:0.0:1788575406.210863:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310769901 00000100:00000001:0.0:1788575406.210865:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310766247 00000100:00000001:0.0:1788575406.210867:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310769901 00000100:00000001:0.0:1788575406.210869:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575406.210871:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.210873:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575406.210876:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575406/real 1788575406] req@ffff8fc4ac821880 x1875440516818816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575406.210883:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575406.210885:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575406.210886:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575406.210888:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575406.210890:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575406.210892:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575406.210895:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516818816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575406.210903:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516818816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210908:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575406.210914:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.210916:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575406.210917:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575406.210922:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516818816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210929:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575406.210932:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516818816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210938:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575406.210941:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.210942:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575406.210943:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.210945:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575406.210947:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575406.210949:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575406.210952:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575406.210954:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575406.210958:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516818816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575406.210966:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575406.210969:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575406.210972:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516818944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.210978:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575406.210979:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575406.210982:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575406.210988:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575406.210990:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575406.210998:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575406.211001:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575406.211003:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575406.211004:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575406.211006:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575406.211009:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575406.211011:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575406.211014:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575406.211016:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575406.211020:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575406.211023:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516818944, portal 4 00000100:00000001:0.0:1788575406.211025:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575406.211026:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788575406.211031:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516818944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.211037:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575406.211039:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575406.211041:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575406.211043:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516818944, offset 0 00000400:00000200:0.0:1788575406.211047:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575406.211051:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575406.211054:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575406.211057:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575406.211059:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575406.211061:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575406.211063:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575406.211066:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548050:548050:256:4294967295] 192.168.202.128@tcp LPNI seq info [547973:547973:8:4294967295] 00000400:00000200:0.0:1788575406.211069:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575406.211075:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575406.211080:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575406.211082:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575406.211091:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575406.211093:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575406.211103:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575406.211105:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575406.211108:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575406.211111:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575406.211114:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575406.211116:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575406.211118:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575406.211122:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516818944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.211129:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575406.211131:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516818944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.211137:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.211138:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575406.211141:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575406.211144:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575406.211147:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575406.211148:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310766249 00000100:00000001:0.0:1788575406.211151:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310769901 00000100:00000001:0.0:1788575406.211152:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310766249 00000100:00000001:0.0:1788575406.211154:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310769901 00000100:00000001:0.0:1788575406.211157:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575406.211158:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.211160:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575406.211163:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575406/real 1788575406] req@ffff8fc4ac820e00 x1875440516818944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575406.211170:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575406.211176:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575406.211177:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575406.211180:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575406.211181:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575406.211184:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575406.211187:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516818944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575406.211195:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516818944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.211201:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575406.211203:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.211204:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575406.211206:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575406.211210:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516818944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.211216:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575406.211220:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516818944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.211226:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575406.211228:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.211230:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575406.211231:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.211233:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575406.211235:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575406.211237:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575406.211240:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575406.211242:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575406.211245:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516818944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575406.211264:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516818944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575406.211268:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.211270:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575406.211273:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516818560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.211279:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575406.211281:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575406.211283:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575406.211285:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575406.211287:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575406.211288:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575406.211290:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575406.211292:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862ac00. 02000000:00000001:0.0:1788575406.211295:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575406.211296:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575406.211298:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788575406.211300:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575406.211301:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.211303:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575406.211306:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516818688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.211312:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575406.211313:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575406.211315:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575406.211317:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575406.211318:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575406.211319:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575406.211321:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575406.211322:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788575406.211324:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575406.211325:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575406.211327:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788575406.211336:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575406.211337:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.211339:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575406.211343:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516818816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.211348:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575406.211350:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575406.211351:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575406.211353:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575406.211355:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575406.211356:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575406.211357:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575406.211359:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788575406.211361:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575406.211362:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575406.211363:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788575406.211365:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575406.211367:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.211368:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575406.211371:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516818944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575406.211376:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575406.211378:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575406.211379:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575406.211381:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575406.211382:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575406.211383:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575406.211384:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575406.211385:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788575406.211388:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575406.211389:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575406.211390:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788575406.211392:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575406.211393:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.211395:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.211398:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575406.211403:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.211405:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575406.211407:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575406.720965:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575406.720971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575406.720975:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575406.720981:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575406.720988:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575406.720990:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.720993:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575406.720994:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575406.720996:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575407.424989:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575407.424992:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575407.424994:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575407.744924:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575407.744928:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575407.744931:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575407.744934:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575407.744938:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575407.744940:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575407.744941:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575407.744942:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575407.744944:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.768896:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575408.768904:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575408.768909:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575408.768916:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575408.768923:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575408.768928:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575408.768934:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575408.768936:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575408.768942:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575408.768969:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575408.768971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575408.768973:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575408.768975:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575408.768976:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575408.768997:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575408.768999:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575408.769000:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575408.769002:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575409.473010:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575409.473013:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575409.473015:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575409.792896:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575409.792902:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575409.792905:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575409.792910:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575409.792916:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575409.792920:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575409.792953:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575409.792956:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575409.792958:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575410.048876:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310769901, at 4310769920 00000100:00000010:0.0:1788575410.048885:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575410.496986:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575410.496990:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575410.496993:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575410.816904:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575410.816912:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575410.816917:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575410.816929:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575410.816934:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575410.816940:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575410.816955:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575410.816959:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575410.816963:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.329139:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575411.329144:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575411.329147:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575411.329155:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516819072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.329172:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575411.329181:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575411.329186:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575411.329191:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575411.329197:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575411.329205:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516819072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575411.329212:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575411.329218:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575411.329223:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575411.329227:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575411.329234:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575411.329238:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575411.329246:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575411.329252:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575411.329260:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575411.329268:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516819072, portal 25 00000100:00000001:0.0:1788575411.329272:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575411.329274:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788575411.329282:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516819072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.329301:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575411.329308:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575411.329313:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575411.329318:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516819072, offset 0 00000400:00000200:0.0:1788575411.329327:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575411.329338:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575411.329348:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575411.329355:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575411.329361:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575411.329367:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575411.329373:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575411.329381:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548051:548051:256:4294967295] 192.168.202.128@tcp LPNI seq info [547974:547974:8:4294967295] 00000400:00000200:0.0:1788575411.329397:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575411.329414:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575411.329428:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575411.329435:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575411.329448:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575411.329452:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575411.329485:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575411.329490:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575411.329497:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575411.329505:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575411.329513:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575411.329517:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575411.329525:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575411.329532:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516819072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.329546:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575411.329551:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516819072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.329565:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.329570:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575411.329577:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575411.329584:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575411.329589:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310775021 00000100:00000010:0.0:1788575411.329596:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575411.329604:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575411.329611:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310775021 00000100:00000010:0.0:1788575411.329617:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b509c0. 00000100:00000001:0.0:1788575411.329621:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575411.329626:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575411.329631:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.329635:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575411.329644:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575411/real 1788575411] req@ffff8fc4ac820e00 x1875440516819072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575411.329683:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575411.329687:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575411.329691:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575411.329695:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575411.329699:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575411.329704:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575411.329711:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516819072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575411.329730:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516819072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.329744:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575411.329746:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.329750:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575411.329752:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575411.329759:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516819072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.329773:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575411.329781:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516819072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.329794:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575411.329798:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.329802:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575411.329803:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.329838:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575411.329842:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575411.329849:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575411.329858:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575411.329862:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575411.329870:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516819072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575411.329895:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516819072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575411.329903:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.329908:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575411.329913:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516819072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.329928:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575411.329932:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575411.329936:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575411.329941:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575411.329945:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575411.329955:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575411.329960:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575411.329964:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788575411.329971:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575411.329975:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575411.329978:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788575411.329984:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575411.329987:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.329991:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330000:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575411.330003:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.330007:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575411.330010:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575411.330012:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575411.330020:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516819200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330031:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575411.330033:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575411.330044:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575411.330048:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575411.330052:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575411.330059:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516819200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575411.330071:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575411.330073:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575411.330076:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575411.330079:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575411.330082:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575411.330084:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575411.330089:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575411.330092:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575411.330095:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575411.330098:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516819200, portal 10 00000100:00000001:0.0:1788575411.330100:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575411.330101:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788575411.330107:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516819200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330114:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575411.330117:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575411.330119:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575411.330121:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516819200, offset 0 00000400:00000200:0.0:1788575411.330125:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575411.330129:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575411.330132:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575411.330135:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575411.330138:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575411.330140:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575411.330143:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575411.330146:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548052:548052:256:4294967295] 192.168.202.128@tcp LPNI seq info [547975:547975:8:4294967295] 00000400:00000200:0.0:1788575411.330150:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575411.330156:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575411.330161:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575411.330164:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575411.330167:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575411.330175:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575411.330186:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575411.330188:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575411.330191:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575411.330195:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575411.330198:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575411.330200:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575411.330203:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575411.330206:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516819200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330214:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575411.330218:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516819200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330224:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.330226:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575411.330229:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575411.330232:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575411.330235:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575411.330238:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310771219 00000100:00000001:0.0:1788575411.330240:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310775021 00000100:00000001:0.0:1788575411.330243:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310771219 00000100:00000001:0.0:1788575411.330245:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310775021 00000100:00000001:0.0:1788575411.330247:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575411.330249:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.330250:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575411.330254:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575411/real 1788575411] req@ffff8fc4ac821880 x1875440516819200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575411.330262:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575411.330263:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575411.330269:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575411.330272:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575411.330273:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575411.330276:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575411.330279:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516819200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575411.330288:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516819200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330295:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575411.330296:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330298:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575411.330299:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575411.330303:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516819200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330310:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575411.330313:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516819200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330320:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575411.330323:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330324:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575411.330326:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330328:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575411.330330:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575411.330332:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575411.330335:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575411.330337:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575411.330341:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516819200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575411.330350:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516819200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575411.330357:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575411.330361:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516819328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330368:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575411.330369:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575411.330371:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575411.330373:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575411.330375:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575411.330378:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516819328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575411.330381:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575411.330383:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575411.330384:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575411.330386:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575411.330389:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575411.330391:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575411.330395:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575411.330397:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575411.330400:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575411.330404:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516819328, portal 10 00000100:00000001:0.0:1788575411.330406:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575411.330407:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788575411.330413:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516819328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330419:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575411.330422:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575411.330424:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575411.330426:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516819328, offset 0 00000400:00000200:0.0:1788575411.330430:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575411.330433:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575411.330437:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575411.330439:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575411.330447:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575411.330449:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575411.330452:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575411.330467:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548053:548053:256:4294967295] 192.168.202.128@tcp LPNI seq info [547976:547976:8:4294967295] 00000400:00000200:0.0:1788575411.330471:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575411.330477:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575411.330482:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575411.330485:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575411.330489:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575411.330491:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575411.330501:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575411.330502:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575411.330505:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575411.330509:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575411.330512:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575411.330514:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575411.330516:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575411.330520:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516819328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330528:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575411.330530:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516819328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330537:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.330538:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575411.330541:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575411.330543:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575411.330545:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575411.330546:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310771226 00000100:00000001:0.0:1788575411.330548:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310775021 00000100:00000001:0.0:1788575411.330549:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310771226 00000100:00000001:0.0:1788575411.330554:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310775021 00000100:00000001:0.0:1788575411.330556:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575411.330557:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.330558:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575411.330561:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575411/real 1788575411] req@ffff8fc4ac823800 x1875440516819328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575411.330566:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575411.330567:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575411.330568:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575411.330569:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575411.330570:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575411.330572:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575411.330574:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516819328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575411.330578:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516819328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330582:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575411.330583:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330584:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575411.330584:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575411.330586:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516819328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330590:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575411.330592:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516819328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330596:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575411.330597:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330598:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575411.330598:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330602:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575411.330603:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575411.330605:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575411.330607:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575411.330608:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575411.330611:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516819328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575411.330616:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516819328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575411.330619:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575411.330622:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516819456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330628:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575411.330629:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575411.330631:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575411.330633:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575411.330635:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575411.330637:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516819456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575411.330640:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575411.330642:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575411.330643:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575411.330645:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575411.330648:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575411.330650:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575411.330654:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575411.330656:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575411.330659:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575411.330662:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516819456, portal 4 00000100:00000001:0.0:1788575411.330664:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575411.330665:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788575411.330671:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516819456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330682:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575411.330684:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575411.330687:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575411.330689:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516819456, offset 0 00000400:00000200:0.0:1788575411.330693:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575411.330697:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575411.330700:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575411.330703:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575411.330706:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575411.330708:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575411.330710:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575411.330713:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548054:548054:256:4294967295] 192.168.202.128@tcp LPNI seq info [547977:547977:8:4294967295] 00000400:00000200:0.0:1788575411.330718:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575411.330724:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575411.330729:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575411.330732:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575411.330735:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575411.330738:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575411.330747:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575411.330749:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575411.330752:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575411.330755:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575411.330758:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575411.330759:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575411.330762:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575411.330766:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516819456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330773:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575411.330777:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516819456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330794:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.330796:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575411.330800:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575411.330803:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575411.330806:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575411.330821:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310771367 00000100:00000001:0.0:1788575411.330822:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310775021 00000100:00000001:0.0:1788575411.330824:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310771367 00000100:00000001:0.0:1788575411.330825:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310775021 00000100:00000001:0.0:1788575411.330826:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575411.330828:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.330829:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575411.330831:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575411/real 1788575411] req@ffff8fc4ac820a80 x1875440516819456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575411.330836:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575411.330837:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575411.330839:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575411.330840:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575411.330841:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575411.330843:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575411.330845:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516819456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575411.330849:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516819456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330853:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575411.330854:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330855:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575411.330856:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575411.330858:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516819456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330865:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575411.330868:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516819456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330872:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575411.330873:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330874:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575411.330875:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330876:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575411.330877:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575411.330878:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575411.330880:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575411.330881:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575411.330883:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516819456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575411.330887:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516819456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575411.330889:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330890:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575411.330892:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516819200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330896:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575411.330897:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575411.330898:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575411.330899:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575411.330900:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575411.330901:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575411.330902:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575411.330903:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788575411.330907:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575411.330908:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575411.330909:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788575411.330910:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575411.330911:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330912:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575411.330914:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516819328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330918:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575411.330918:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575411.330920:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575411.330921:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575411.330921:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575411.330922:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575411.330923:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575411.330924:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788575411.330925:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575411.330925:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575411.330926:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788575411.330927:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575411.330928:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330929:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575411.330930:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516819456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330933:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575411.330934:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575411.330935:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575411.330936:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575411.330936:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575411.330937:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575411.330938:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575411.330939:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788575411.330939:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575411.330940:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575411.330941:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788575411.330942:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575411.330945:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330946:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.330948:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575411.330949:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.330950:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575411.330951:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575411.330952:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575411.330954:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516819584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330958:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575411.330959:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575411.330960:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575411.330962:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575411.330963:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575411.330964:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516819584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575411.330966:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575411.330967:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575411.330968:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575411.330969:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575411.330970:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575411.330972:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575411.330974:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575411.330975:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575411.330977:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575411.330979:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516819584, portal 4 00000100:00000001:0.0:1788575411.330980:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575411.330981:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788575411.330984:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516819584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.330988:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575411.330989:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575411.330990:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575411.330992:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516819584, offset 0 00000400:00000200:0.0:1788575411.330994:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575411.330999:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575411.331001:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575411.331003:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575411.331005:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575411.331006:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575411.331008:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575411.331009:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548055:548055:256:4294967295] 192.168.202.128@tcp LPNI seq info [547978:547978:8:4294967295] 00000400:00000200:0.0:1788575411.331012:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575411.331016:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575411.331018:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575411.331020:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575411.331022:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575411.331024:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575411.331030:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575411.331031:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575411.331033:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575411.331035:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575411.331037:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575411.331038:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575411.331039:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575411.331041:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516819584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.331046:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575411.331047:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516819584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.331051:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.331052:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575411.331053:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575411.331055:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575411.331057:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575411.331061:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310771369 00000100:00000001:0.0:1788575411.331062:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310775021 00000100:00000001:0.0:1788575411.331064:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310771369 00000100:00000001:0.0:1788575411.331065:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310775021 00000100:00000001:0.0:1788575411.331066:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575411.331067:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.331068:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575411.331071:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575411/real 1788575411] req@ffff8fc4ac821c00 x1875440516819584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575411.331080:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575411.331081:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575411.331082:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575411.331083:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575411.331084:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575411.331086:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575411.331087:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516819584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575411.331092:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516819584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.331096:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575411.331097:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.331097:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575411.331098:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575411.331100:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516819584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.331103:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575411.331105:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516819584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.331109:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575411.331113:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.331114:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575411.331114:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.331115:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575411.331117:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575411.331118:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575411.331120:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575411.331121:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575411.331123:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516819584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575411.331128:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516819584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575411.331130:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.331131:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575411.331133:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516819584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575411.331136:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575411.331137:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575411.331138:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575411.331139:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575411.331140:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575411.331140:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575411.331141:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575411.331142:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788575411.331143:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575411.331144:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575411.331145:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788575411.331146:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575411.331147:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.331148:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.331150:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575411.331151:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.331152:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575411.331156:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575411.840881:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575411.840888:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575411.840891:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575411.840897:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575411.840904:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575411.840910:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.840921:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575411.840924:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575411.840928:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575412.545005:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575412.545009:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575412.545011:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575412.864962:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575412.864983:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575412.864990:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575412.865000:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575412.865008:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575412.865018:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575412.865030:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575412.865034:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575412.865038:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575413.888884:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575413.888892:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575413.888896:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575413.888904:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575413.888909:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575413.888912:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575413.888914:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575413.888915:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575413.888919:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575413.888964:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575413.888969:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575413.888970:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575413.888973:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575413.888978:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575413.888982:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575413.888987:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575413.888996:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575413.889009:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575414.593035:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575414.593038:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575414.593041:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575414.912940:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575414.912948:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575414.912951:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575414.912966:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575414.912976:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575414.912987:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575414.913009:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575414.913012:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575414.913014:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575415.168939:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310775021, at 4310775040 00000100:00000010:0.0:1788575415.168958:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575415.617073:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575415.617079:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575415.617082:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575415.936921:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575415.936925:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575415.936928:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575415.936932:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575415.936939:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575415.936943:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575415.936946:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575415.936949:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575415.936953:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575416.448924:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575416.448932:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575416.448935:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575416.448938:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575416.448942:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575416.448948:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16103 00000400:00000200:0.0:1788575416.448957:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575416.448978:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575416.448981:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575416.448983:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.448985:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575416.448987:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.448989:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575416.448991:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575416.448993:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575416.448994:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575416.448997:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575416.448998:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575416.449000:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575416.449001:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449002:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575416.449004:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449005:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575416.449006:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575416.449009:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449010:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575416.449012:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575416.449015:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575416.449017:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449018:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575416.449020:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575416.449024:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575416.449026:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575416.449029:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575416.449031:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449034:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575416.449035:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575416.449038:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575416.449043:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575416.449058:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575416.449065:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516819712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575416.449072:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575416.449075:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575416.449078:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516819712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575416.449093:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575416.449096:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575416.449099:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575416.449101:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575416.449102:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575416.449105:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575416.449109:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16103 00000400:00000200:0.0:1788575416.449116:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575416.449122:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575416.449126:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575416.449132:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449138:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575416.449142:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449145:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575416.449148:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575416.449151:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575416.449154:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575416.449157:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575416.449160:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575416.449164:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575416.449169:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449172:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575416.449181:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449185:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575416.449192:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575416.449204:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449212:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575416.449217:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575416.449225:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575416.449230:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449236:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575416.449241:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575416.449252:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449263:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575416.449270:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575416.449276:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575416.449281:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575416.449284:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449288:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575416.449291:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575416.449295:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575416.449302:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575416.449308:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575416.449317:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516819840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575416.449336:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575416.449340:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575416.449349:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516819840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575416.449365:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575416.449370:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575416.449376:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575416.449378:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575416.449386:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575416.449391:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575416.449395:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16103 00000400:00000200:0.0:1788575416.449402:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575416.449438:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575416.449443:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575416.449457:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449462:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575416.449467:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449472:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575416.449477:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575416.449479:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575416.449482:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575416.449485:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575416.449488:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575416.449491:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575416.449496:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449500:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575416.449503:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449506:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575416.449509:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575416.449513:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449515:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575416.449516:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575416.449520:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575416.449522:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449524:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575416.449528:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575416.449531:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449533:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575416.449542:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575416.449546:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575416.449550:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575416.449552:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449555:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575416.449557:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575416.449559:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575416.449564:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575416.449569:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575416.449577:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516819968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575416.449598:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575416.449602:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575416.449609:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516819968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575416.449618:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575416.449622:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575416.449624:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575416.449627:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575416.449628:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575416.449634:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575416.449638:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16103 00000400:00000200:0.0:1788575416.449642:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575416.449646:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575416.449649:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575416.449654:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449657:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575416.449659:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449662:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575416.449664:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575416.449669:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575416.449671:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575416.449673:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575416.449677:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575416.449680:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575416.449685:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449688:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575416.449691:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449695:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575416.449698:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575416.449702:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449706:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575416.449709:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575416.449712:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575416.449716:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449718:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575416.449721:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575416.449724:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449727:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575416.449731:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575416.449734:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575416.449737:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575416.449740:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449742:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575416.449744:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575416.449747:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575416.449752:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575416.449755:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575416.449762:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516820096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575416.449772:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575416.449774:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575416.449784:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516820096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575416.449795:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575416.449798:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575416.449804:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575416.449805:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575416.449806:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575416.449831:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575416.449835:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16103 00000400:00000200:0.0:1788575416.449841:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575416.449848:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575416.449853:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575416.449861:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449866:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575416.449872:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449879:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575416.449884:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575416.449888:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575416.449893:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575416.449897:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575416.449902:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575416.449907:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575416.449914:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449919:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575416.449924:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449929:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575416.449935:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575416.449943:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575416.449949:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575416.449958:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575416.449963:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575416.449967:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449970:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575416.449973:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575416.449978:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449981:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575416.449984:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575416.449988:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575416.449989:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575416.449990:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575416.449991:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575416.449992:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575416.449994:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575416.449998:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575416.450003:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575416.450012:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516820224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575416.450021:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575416.450025:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575416.450031:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516820224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575416.450044:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575416.450048:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16114) 00000100:00000001:0.0:1788575416.960908:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575416.960916:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575416.960921:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575416.960930:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575416.960940:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575416.960943:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575416.960965:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575416.960968:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575416.960974:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575417.665008:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575417.665037:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575417.665041:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575417.984920:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575417.984928:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575417.984930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575417.984936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575417.984943:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575417.984949:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575417.984955:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575417.984966:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575417.984970:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575417.985010:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575417.985016:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575417.985018:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575417.985022:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575417.985027:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575417.985030:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575417.985037:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575417.985039:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575417.985041:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575419.008949:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575419.008954:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575419.008959:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575419.008968:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575419.008977:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575419.008985:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575419.009008:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575419.009012:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575419.009016:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575419.713074:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575419.713077:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575419.713080:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575420.032919:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575420.032927:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575420.032933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575420.032941:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575420.032949:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575420.032956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575420.032989:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575420.032992:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575420.032997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575420.288896:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310780141, at 4310780160 00000100:00000010:0.0:1788575420.288909:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf960. 00000020:00000001:0.0:1788575420.737016:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575420.737019:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575420.737021:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575421.056920:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575421.056928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575421.056934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575421.056942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575421.056950:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575421.056956:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575421.056961:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575421.056963:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575421.056966:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575422.080929:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575422.080935:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575422.080940:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575422.080949:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575422.080957:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575422.080971:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575422.080992:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575422.080995:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575422.080998:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575422.785034:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575422.785039:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575422.785043:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575423.104889:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575423.104897:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575423.104903:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575423.104911:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575423.104918:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575423.104923:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575423.104940:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575423.104943:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575423.104956:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575423.104995:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575423.104999:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575423.105001:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575423.105004:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575423.105007:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575423.105009:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575423.105011:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575423.105013:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575423.105015:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.128926:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575424.128932:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575424.128936:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575424.128943:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575424.128951:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575424.128956:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575424.128963:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575424.128969:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575424.128975:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575424.129013:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575424.129020:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575424.129027:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575424.129031:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575424.129036:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575424.129041:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575424.129047:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575424.129050:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575424.129054:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575424.832999:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575424.833002:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575424.833005:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575425.152905:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575425.152911:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575425.152916:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575425.152923:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575425.152928:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575425.152933:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575425.152939:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575425.152951:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575425.152955:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575425.152975:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575425.152980:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575425.152982:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575425.152986:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575425.152989:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575425.152991:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575425.152994:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575425.152996:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575425.152998:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575425.408896:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310785261, at 4310785280 00000100:00000010:0.0:1788575425.408916:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf360. 00000020:00000001:0.0:1788575425.856982:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575425.856986:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575425.856988:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575426.176943:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575426.176952:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575426.176955:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575426.176963:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575426.176968:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575426.176974:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575426.176988:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575426.176993:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575426.176998:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575426.177019:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575426.177024:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575426.177026:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575426.177029:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575426.177032:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575426.177034:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575426.177037:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575426.177039:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575426.177042:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575427.201028:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575427.201032:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575427.201036:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575427.201054:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575427.201081:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575427.201087:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575427.201089:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575427.201090:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575427.201094:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575427.904960:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575427.904965:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575427.904978:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575428.224916:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575428.224924:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575428.224929:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575428.224937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575428.224946:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575428.224954:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575428.224969:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575428.224973:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575428.224976:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575429.248947:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575429.248951:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575429.248957:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575429.248965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575429.248971:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575429.248975:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575429.248990:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575429.248993:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575429.248999:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575429.953040:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575429.953045:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575429.953048:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575430.272915:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575430.272923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575430.272926:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575430.272932:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575430.272938:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575430.272944:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575430.272951:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575430.272955:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575430.272982:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575430.528920:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310790381, at 4310790400 00000100:00000010:0.0:1788575430.528932:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cff60. 00000020:00000001:0.0:1788575430.977007:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575430.977010:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575430.977012:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575431.296901:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575431.296910:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575431.296914:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575431.296923:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575431.296931:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575431.296938:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575431.296946:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575431.296950:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575431.296955:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575431.296991:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575431.297006:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575431.297008:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575431.297012:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575431.297015:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575431.297018:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575431.297020:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575431.297022:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575431.297024:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575432.320931:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575432.320936:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575432.320940:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575432.320945:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575432.320950:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575432.320955:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575432.320966:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575432.320970:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575432.320978:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575433.025086:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575433.025090:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575433.025092:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575433.344961:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575433.344987:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575433.344996:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575433.345012:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575433.345035:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575433.345048:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575433.345064:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575433.345068:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575433.345073:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575434.368974:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575434.368987:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575434.368997:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575434.369016:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575434.369035:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575434.369052:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575434.369082:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575434.369090:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575434.369100:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575434.369172:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575434.369181:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575434.369182:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575434.369184:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575434.369186:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575434.369189:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575434.369192:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575434.369194:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575434.369197:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788575434.881170:0:2389:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788575434.881220:0:2389:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788575434.881231:0:2389:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575434.881247:0:2389:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788575434.881253:0:2389:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575434.881259:0:2389:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788575434.881265:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575434.881271:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793500. 00000400:00000200:0.0:1788575434.881279:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575434.881285:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575434.881313:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcb28 00000400:00000010:0.0:1788575434.881317:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcb28. 00000100:00000001:0.0:1788575434.881324:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575434.881336:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac820000 x1875440516821632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575434.881385:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575434.881390:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516821632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575434.881399:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575434.881401:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575434.881414:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575434.881417:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793d00. 00000400:00000200:0.0:1788575434.881420:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575434.881424:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575434.881427:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc880 00000400:00000010:0.0:1788575434.881429:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc880. 00000100:00000001:0.0:1788575434.881431:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575434.881436:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac823100 x1875440516821760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575434.881444:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575434.881447:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516821760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575434.881454:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575434.881455:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575434.881460:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575434.881463:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793400. 00000400:00000200:0.0:1788575434.881466:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575434.881469:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575434.881472:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc550 00000400:00000010:0.0:1788575434.881474:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc550. 00000100:00000001:0.0:1788575434.881476:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575434.881481:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac822a00 x1875440516821888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575434.881496:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575434.881500:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516821888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575434.881506:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575434.881507:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575434.881514:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575434.881517:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793b00. 00000400:00000200:0.0:1788575434.881521:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575434.881527:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575434.881531:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fca18 00000400:00000010:0.0:1788575434.881535:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fca18. 00000100:00000001:0.0:1788575434.881538:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575434.881544:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac823b80 x1875440516822016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575434.881554:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575434.881560:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516822016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575434.881581:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575434.881584:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575434.881595:0:2389:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575434.881599:0:2389:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793000. 00000400:00000200:0.0:1788575434.881607:0:2389:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575434.881621:0:2389:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575434.881632:0:2389:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc3b8 00000400:00000010:0.0:1788575434.881638:0:2389:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc3b8. 00000100:00000001:0.0:1788575434.881651:0:2389:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575434.881658:0:2389:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4ac821500 x1875440516822144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575434.881677:0:2389:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575434.881685:0:2389:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516822144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575434.881728:0:2389:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575434.881733:0:2389:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788575435.073073:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575435.073078:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575435.073082:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575435.392966:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575435.392976:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575435.392982:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575435.392990:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575435.393000:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575435.393005:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575435.393029:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575435.393040:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575435.393048:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575435.393072:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575435.393081:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575435.393086:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575435.393092:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575435.393096:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575435.393100:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575435.393103:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575435.393106:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575435.393111:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575435.648955:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310795501, at 4310795520 00000100:00000010:0.0:1788575435.648976:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf180. 00000020:00000001:0.0:1788575436.097034:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575436.097038:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575436.097041:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575436.416927:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575436.416937:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575436.416941:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575436.416952:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575436.416963:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575436.416977:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575436.417003:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575436.417028:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575436.417032:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575437.440957:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575437.440971:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575437.440976:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575437.440988:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575437.441001:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575437.441014:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575437.441027:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575437.441031:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575437.441037:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575438.145030:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575438.145034:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575438.145036:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575438.464935:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575438.464947:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575438.464950:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575438.464959:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575438.464970:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575438.464985:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575438.464992:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575438.465000:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575438.465006:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575439.488911:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575439.488918:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575439.488920:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575439.488926:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575439.488931:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575439.488935:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575439.488940:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575439.488944:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575439.488949:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575439.488994:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575439.488997:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575439.488998:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575439.489000:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575439.489003:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575439.489005:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575439.489024:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575439.489027:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575439.489034:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575440.193101:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575440.193108:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575440.193113:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575440.512905:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575440.512914:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575440.512919:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575440.512928:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575440.512937:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575440.512943:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575440.512964:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575440.512969:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575440.512975:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575440.769759:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310800622, at 4310800640 00000100:00000010:0.0:1788575440.769768:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfb40. 00000020:00000001:0.0:1788575441.217019:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575441.217023:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575441.217027:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575441.536922:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575441.536931:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575441.536935:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575441.536941:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575441.536948:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575441.536954:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575441.536960:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575441.536964:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575441.536969:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.536988:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575441.536991:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575441.536993:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575441.536995:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575441.536998:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575441.536999:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575441.537001:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575441.537003:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575441.537012:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575441.537040:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575441.537044:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575441.537047:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575441.537052:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575441.537057:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575441.537060:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575441.537062:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575441.537063:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575441.537066:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575442.048930:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575442.048941:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575442.048944:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575442.048947:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575442.048952:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575442.048960:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16129 00000400:00000200:0.0:1788575442.048972:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575442.048979:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575442.048983:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575442.048987:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.048990:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575442.048993:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.048995:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575442.049000:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575442.049002:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575442.049005:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575442.049009:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575442.049012:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575442.049015:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575442.049018:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049020:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575442.049022:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049038:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575442.049041:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575442.049044:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049046:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575442.049048:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575442.049051:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575442.049053:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.049055:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575442.049057:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575442.049063:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575442.049067:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575442.049071:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575442.049073:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.049076:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575442.049079:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575442.049084:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575442.049091:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575442.049098:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575442.049107:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516822912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575442.049118:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575442.049123:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575442.049129:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516822912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575442.049152:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575442.049156:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575442.049159:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575442.049161:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575442.049163:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575442.049166:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575442.049183:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16129 00000400:00000200:0.0:1788575442.049194:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575442.049199:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575442.049202:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575442.049206:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049209:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575442.049214:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049217:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575442.049220:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575442.049221:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575442.049223:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575442.049226:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575442.049230:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575442.049235:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575442.049242:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049247:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575442.049252:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049254:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575442.049257:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575442.049263:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049270:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575442.049275:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575442.049283:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575442.049290:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.049296:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575442.049302:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575442.049310:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.049315:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575442.049323:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575442.049329:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575442.049334:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575442.049344:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.049348:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575442.049350:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575442.049355:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575442.049362:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575442.049368:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575442.049380:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516823040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575442.049397:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575442.049403:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575442.049411:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516823040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575442.049432:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575442.049439:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575442.049444:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575442.049447:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575442.049451:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575442.049455:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575442.049461:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16129 00000400:00000200:0.0:1788575442.049472:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575442.049482:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575442.049486:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575442.049494:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049501:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575442.049507:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049513:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575442.049520:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575442.049524:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575442.049529:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575442.049534:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575442.049541:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575442.049557:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575442.049567:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049571:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575442.049576:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049584:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575442.049590:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575442.049599:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049605:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575442.049611:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575442.049617:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575442.049624:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.049630:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575442.049635:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575442.049643:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.049647:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575442.049652:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575442.049656:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575442.049658:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575442.049659:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.049662:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575442.049665:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575442.049668:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575442.049673:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575442.049678:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575442.049683:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516823168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575442.049691:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575442.049695:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575442.049700:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516823168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575442.049711:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575442.049719:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575442.049725:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575442.049730:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575442.049733:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575442.049740:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575442.049744:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16129 00000400:00000200:0.0:1788575442.049750:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575442.049756:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575442.049759:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575442.049765:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049769:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575442.049775:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049780:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575442.049784:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575442.049786:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575442.049789:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575442.049791:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575442.049797:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575442.049802:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575442.049849:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049854:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575442.049860:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049865:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575442.049871:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575442.049878:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575442.049882:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575442.049888:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575442.049892:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575442.049897:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.049902:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575442.049912:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575442.049917:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.049922:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575442.049931:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575442.049938:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575442.049943:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575442.049947:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.049953:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575442.049958:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575442.049962:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575442.049970:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575442.049978:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575442.049991:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516823296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575442.050008:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575442.050014:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575442.050021:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516823296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575442.050036:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575442.050042:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575442.050049:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575442.050050:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575442.050052:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575442.050057:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575442.050062:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16129 00000400:00000200:0.0:1788575442.050067:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575442.050069:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575442.050071:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575442.050075:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.050080:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575442.050085:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.050095:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575442.050098:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575442.050101:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575442.050104:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575442.050108:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575442.050112:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575442.050118:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575442.050127:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.050130:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575442.050135:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575442.050140:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575442.050145:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575442.050150:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575442.050156:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575442.050162:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575442.050167:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575442.050183:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.050187:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575442.050190:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575442.050196:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.050200:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575442.050207:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575442.050213:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575442.050218:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575442.050222:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.050227:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575442.050228:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575442.050231:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575442.050237:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575442.050243:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575442.050252:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516823424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575442.050279:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575442.050283:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575442.050291:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516823424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575442.050304:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575442.050309:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16139) 00000100:00000001:0.0:1788575442.560886:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575442.560893:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575442.560898:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575442.560905:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575442.560910:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575442.560915:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575442.560919:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575442.560923:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575442.560927:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575442.560958:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575442.560964:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575442.560967:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575442.560972:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575442.560977:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575442.560982:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575442.560985:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575442.560986:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575442.560989:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575443.265027:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575443.265030:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575443.265032:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575443.584907:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575443.584914:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575443.584918:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575443.584927:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575443.584939:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575443.584945:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575443.584961:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575443.584978:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575443.584981:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575444.608914:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575444.608921:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575444.608924:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575444.608932:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575444.608940:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575444.608945:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575444.609115:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575444.609117:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575444.609120:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575445.313013:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575445.313016:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575445.313018:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575445.632887:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575445.632892:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575445.632896:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575445.632902:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575445.632907:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575445.632913:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575445.632918:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575445.632922:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575445.632929:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575445.632947:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575445.632951:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575445.632955:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575445.632959:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575445.632961:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575445.632963:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575445.632965:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575445.632966:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575445.632968:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575445.888908:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310805741, at 4310805760 00000100:00000010:0.0:1788575445.888924:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf540. 00000020:00000001:0.0:1788575446.336995:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575446.336999:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575446.337001:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575446.656941:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575446.656946:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575446.656949:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575446.656953:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575446.656959:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575446.656963:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575446.656969:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575446.656974:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575446.656979:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575447.681020:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575447.681024:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575447.681025:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575447.681028:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575447.681031:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575447.681033:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575447.681036:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575447.681038:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575447.681040:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575448.385030:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575448.385033:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575448.385035:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575449.728915:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575449.728920:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575449.728923:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575449.728930:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575449.728937:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575449.728941:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575449.728948:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575449.728951:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575449.728955:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575450.432996:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575450.432999:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575450.433002:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575450.752900:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575450.752909:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575450.752913:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575450.752919:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575450.752926:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575450.752945:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575450.752952:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575450.752954:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575450.752957:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575451.008916:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310810861, at 4310810880 00000100:00000010:0.0:1788575451.008930:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf3c0. 00000020:00000001:0.0:1788575451.457000:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575451.457003:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575451.457005:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575451.776913:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575451.776920:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575451.776922:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575451.776929:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575451.776933:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575451.776938:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575451.776943:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575451.776946:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575451.776951:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575451.776980:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575451.776983:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575451.776984:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575451.776986:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575451.776988:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575451.776990:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575451.776991:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575451.776992:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575451.776993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.289153:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575452.289159:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575452.289162:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575452.289171:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516824192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.289184:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575452.289186:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575452.289190:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575452.289194:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575452.289198:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575452.289215:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516824192:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575452.289221:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575452.289225:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575452.289228:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575452.289233:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575452.289239:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575452.289243:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575452.289253:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575452.289259:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575452.289268:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575452.289277:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516824192, portal 25 00000100:00000001:0.0:1788575452.289280:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575452.289284:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575452.289290:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516824192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.289299:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575452.289305:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575452.289310:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575452.289315:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516824192, offset 0 00000400:00000200:0.0:1788575452.289324:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575452.289335:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575452.289344:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575452.289353:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575452.289361:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575452.289367:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575452.289374:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575452.289382:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548091:548091:256:4294967295] 192.168.202.128@tcp LPNI seq info [548014:548014:8:4294967295] 00000400:00000200:0.0:1788575452.289391:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575452.289407:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575452.289415:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575452.289422:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575452.289437:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575452.289441:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575452.289457:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575452.289461:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575452.289466:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575452.289471:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575452.289477:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575452.289480:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575452.289484:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575452.289490:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516824192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.289504:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575452.289510:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516824192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.289522:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.289525:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575452.289531:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575452.289537:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575452.289541:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310815981 00000100:00000010:0.0:1788575452.289547:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575452.289555:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575452.289561:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310815981 00000100:00000010:0.0:1788575452.289567:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50180. 00000100:00000001:0.0:1788575452.289571:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575452.289575:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575452.289579:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.289583:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575452.289589:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575452/real 1788575452] req@ffff8fc4901a4a80 x1875440516824192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575452.289607:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575452.289615:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575452.289621:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575452.289626:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575452.289630:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575452.289636:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575452.289644:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516824192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575452.289665:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516824192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.289682:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575452.289686:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.289691:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575452.289695:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575452.289705:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516824192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.289722:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575452.289731:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516824192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.289751:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575452.289759:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.289764:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575452.289768:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.289774:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575452.289777:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575452.289784:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575452.289792:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575452.289798:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575452.289805:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516824192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575452.289872:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516824192:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575452.289882:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.289883:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575452.289887:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516824192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.289892:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575452.289894:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575452.289896:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575452.289899:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575452.289900:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575452.289902:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575452.289904:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575452.289906:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575452.289910:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575452.289911:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575452.289913:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575452.289917:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575452.289918:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.289919:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.289924:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575452.289926:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.289928:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575452.289929:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575452.289930:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575452.289932:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516824320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.289936:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575452.289937:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575452.289939:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575452.289940:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575452.289941:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575452.289943:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516824320:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575452.289945:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575452.289946:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575452.289947:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575452.289951:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575452.289953:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575452.289954:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575452.289957:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575452.289959:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575452.289961:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575452.289963:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516824320, portal 10 00000100:00000001:0.0:1788575452.289964:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575452.289965:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575452.289969:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516824320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.289973:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575452.289975:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575452.289976:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575452.289977:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516824320, offset 0 00000400:00000200:0.0:1788575452.289980:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575452.289982:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575452.289984:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575452.289986:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575452.289988:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575452.289989:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575452.289991:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575452.289993:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548092:548092:256:4294967295] 192.168.202.128@tcp LPNI seq info [548015:548015:8:4294967295] 00000400:00000200:0.0:1788575452.289995:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575452.289999:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575452.290002:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575452.290004:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575452.290007:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575452.290008:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575452.290015:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575452.290016:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575452.290021:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575452.290023:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575452.290026:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575452.290027:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575452.290028:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575452.290030:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516824320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290035:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575452.290037:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516824320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290041:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.290042:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575452.290044:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575452.290046:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575452.290048:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575452.290050:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310812179 00000100:00000001:0.0:1788575452.290051:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310815981 00000100:00000001:0.0:1788575452.290053:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310812179 00000100:00000001:0.0:1788575452.290054:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310815981 00000100:00000001:0.0:1788575452.290055:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575452.290056:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.290057:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575452.290059:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575452/real 1788575452] req@ffff8fc4901a7100 x1875440516824320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575452.290064:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575452.290065:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575452.290066:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575452.290067:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575452.290068:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575452.290069:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575452.290090:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516824320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575452.290095:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516824320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290099:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575452.290100:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.290101:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575452.290102:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575452.290104:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516824320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290108:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575452.290110:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516824320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290114:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575452.290115:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.290116:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575452.290117:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.290118:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575452.290119:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575452.290121:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575452.290122:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575452.290123:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575452.290125:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516824320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575452.290130:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516824320:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575452.290132:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575452.290134:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516824448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290144:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575452.290145:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575452.290147:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575452.290148:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575452.290149:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575452.290151:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516824448:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575452.290152:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575452.290154:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575452.290155:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575452.290156:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575452.290158:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575452.290159:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575452.290161:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575452.290162:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575452.290164:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575452.290166:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516824448, portal 10 00000100:00000001:0.0:1788575452.290167:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575452.290168:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575452.290172:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516824448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290175:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575452.290177:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575452.290178:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575452.290180:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516824448, offset 0 00000400:00000200:0.0:1788575452.290182:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575452.290184:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575452.290186:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575452.290188:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575452.290189:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575452.290191:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575452.290192:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575452.290197:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548093:548093:256:4294967295] 192.168.202.128@tcp LPNI seq info [548016:548016:8:4294967295] 00000400:00000200:0.0:1788575452.290199:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575452.290203:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575452.290206:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575452.290208:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575452.290210:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575452.290212:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575452.290218:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575452.290219:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575452.290221:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575452.290223:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575452.290225:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575452.290226:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575452.290229:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575452.290232:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516824448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290239:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575452.290242:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516824448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290248:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.290249:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575452.290252:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575452.290255:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575452.290258:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575452.290260:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310812186 00000100:00000001:0.0:1788575452.290263:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310815981 00000100:00000001:0.0:1788575452.290265:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310812186 00000100:00000001:0.0:1788575452.290267:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310815981 00000100:00000001:0.0:1788575452.290268:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575452.290275:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.290276:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575452.290279:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575452/real 1788575452] req@ffff8fc4901a5180 x1875440516824448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575452.290285:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575452.290286:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575452.290287:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575452.290289:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575452.290290:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575452.290291:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575452.290293:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516824448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575452.290298:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516824448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290301:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575452.290302:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.290304:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575452.290305:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575452.290308:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516824448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290314:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575452.290318:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516824448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290324:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575452.290326:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.290327:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575452.290329:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.290330:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575452.290333:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575452.290336:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575452.290344:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575452.290347:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575452.290351:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516824448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575452.290358:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516824448:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575452.290361:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575452.290365:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516824576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290371:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575452.290372:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575452.290375:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575452.290376:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575452.290378:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575452.290381:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516824576:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575452.290384:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575452.290386:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575452.290387:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575452.290389:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575452.290392:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575452.290394:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575452.290398:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575452.290400:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575452.290404:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575452.290406:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516824576, portal 4 00000100:00000001:0.0:1788575452.290409:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575452.290410:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575452.290415:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516824576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290421:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575452.290429:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575452.290431:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575452.290434:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516824576, offset 0 00000400:00000200:0.0:1788575452.290438:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575452.290442:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575452.290446:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575452.290448:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575452.290450:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575452.290452:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575452.290454:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575452.290457:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548094:548094:256:4294967295] 192.168.202.128@tcp LPNI seq info [548017:548017:8:4294967295] 00000400:00000200:0.0:1788575452.290461:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575452.290467:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575452.290471:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575452.290474:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575452.290478:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575452.290481:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575452.290496:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575452.290498:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575452.290501:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575452.290505:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575452.290508:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575452.290510:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575452.290513:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575452.290516:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516824576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290523:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575452.290526:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516824576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290532:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.290534:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575452.290547:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575452.290550:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575452.290554:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575452.290556:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310812326 00000100:00000001:0.0:1788575452.290558:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310815981 00000100:00000001:0.0:1788575452.290560:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310812326 00000100:00000001:0.0:1788575452.290562:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310815981 00000100:00000001:0.0:1788575452.290564:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575452.290565:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.290567:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575452.290571:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575452/real 1788575452] req@ffff8fc4901a5c00 x1875440516824576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575452.290579:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575452.290581:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575452.290582:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575452.290585:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575452.290586:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575452.290589:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575452.290591:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516824576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575452.290600:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516824576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290606:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575452.290607:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.290609:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575452.290610:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575452.290613:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516824576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290619:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575452.290622:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516824576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290635:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575452.290637:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.290639:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575452.290641:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.290643:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575452.290644:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575452.290647:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575452.290650:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575452.290653:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575452.290657:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516824576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575452.290664:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516824576:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575452.290668:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575452.290671:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516824704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290677:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575452.290679:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575452.290681:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575452.290683:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575452.290685:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575452.290688:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516824704:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575452.290691:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575452.290693:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575452.290694:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575452.290696:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575452.290700:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575452.290702:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575452.290706:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575452.290766:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575452.290770:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575452.290773:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516824704, portal 4 00000100:00000001:0.0:1788575452.290775:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575452.290776:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575452.290783:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516824704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290789:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575452.290792:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575452.290794:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575452.290797:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516824704, offset 0 00000400:00000200:0.0:1788575452.290801:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575452.290805:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575452.290831:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575452.290835:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575452.290839:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575452.290841:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575452.290845:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575452.290849:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548095:548095:256:4294967295] 192.168.202.128@tcp LPNI seq info [548018:548018:8:4294967295] 00000400:00000200:0.0:1788575452.290853:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575452.290859:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575452.290863:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575452.290867:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575452.290870:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575452.290873:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575452.290883:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575452.290885:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575452.290888:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575452.290891:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575452.290894:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575452.290896:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575452.290904:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575452.290908:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516824704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290915:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575452.290919:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516824704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575477 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.290925:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.290926:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575452.290930:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575452.290933:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575452.290937:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575452.290939:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310812329 00000100:00000001:0.0:1788575452.290941:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310815981 00000100:00000001:0.0:1788575452.290943:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310812329 00000100:00000001:0.0:1788575452.290945:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310815981 00000100:00000001:0.0:1788575452.290947:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575452.290949:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.290951:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575452.290955:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575452/real 1788575452] req@ffff8fc4901a7800 x1875440516824704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575452.290963:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575452.290965:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575452.290966:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575452.290968:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575452.290970:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575452.290972:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575452.290975:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516824704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575452.290983:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516824704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.291000:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575452.291002:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.291004:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575452.291005:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575452.291009:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516824704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.291016:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575452.291020:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516824704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.291026:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575452.291028:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.291029:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575452.291031:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.291033:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575452.291034:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575452.291037:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575452.291041:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575452.291043:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575452.291047:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516824704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575452.291054:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516824704:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575452.291057:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.291059:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575452.291063:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516824320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.291070:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575452.291071:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575452.291091:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575452.291094:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575452.291096:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575452.291097:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575452.291099:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575452.291101:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575452.291104:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575452.291105:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575452.291107:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575452.291110:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575452.291112:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.291114:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575452.291119:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516824448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.291125:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575452.291126:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575452.291128:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575452.291130:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575452.291131:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575452.291132:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575452.291134:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575452.291135:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575452.291138:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575452.291139:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575452.291140:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575452.291144:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575452.291145:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.291147:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575452.291150:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516824576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.291156:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575452.291157:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575452.291159:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575452.291162:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575452.291163:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575452.291164:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575452.291171:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575452.291173:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575452.291175:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575452.291177:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575452.291178:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575452.291180:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575452.291181:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.291183:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575452.291187:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516824704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575477 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575452.291192:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575452.291194:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575452.291195:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575452.291197:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575452.291198:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575452.291199:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575452.291201:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575452.291202:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575452.291205:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575452.291206:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575452.291207:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575452.291210:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575452.291211:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.291213:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.291216:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575452.291218:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.291219:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575452.291221:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575452.800904:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575452.800912:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575452.800914:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575452.800919:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575452.800926:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575452.800930:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.800948:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575452.800951:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575452.800954:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575453.504973:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575453.504977:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575453.504981:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575453.824914:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575453.824920:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575453.824923:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575453.824929:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575453.824933:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575453.824937:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575453.824941:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575453.824943:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575453.824947:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575454.848917:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575454.848924:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575454.848928:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575454.848935:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575454.848939:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575454.848942:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575454.848946:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575454.848950:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575454.848954:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788575455.361120:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575455.361136:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575455.361142:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575455.361145:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788575455.552949:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575455.552952:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575455.552954:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575455.872909:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575455.872916:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575455.872920:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575455.872926:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575455.872931:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575455.872937:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575455.872952:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575455.872955:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575455.872958:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575456.128917:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310815981, at 4310816000 00000100:00000010:0.0:1788575456.128943:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575456.577000:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575456.577004:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575456.577006:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575456.897077:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575456.897080:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575456.897081:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575456.897083:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575456.897085:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575456.897086:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575456.897087:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575456.897088:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575456.897089:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.409145:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575457.409151:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575457.409154:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575457.409161:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516824832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.409172:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575457.409175:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575457.409180:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575457.409184:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575457.409189:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575457.409196:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516824832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575457.409202:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575457.409206:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575457.409208:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575457.409211:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575457.409217:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575457.409222:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575457.409230:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575457.409233:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575457.409239:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575457.409246:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516824832, portal 25 00000100:00000001:0.0:1788575457.409250:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575457.409263:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575457.409270:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516824832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.409281:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575457.409284:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575457.409289:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575457.409293:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516824832, offset 0 00000400:00000200:0.0:1788575457.409301:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575457.409309:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575457.409317:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575457.409323:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575457.409328:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575457.409332:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575457.409336:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575457.409342:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548096:548096:256:4294967295] 192.168.202.128@tcp LPNI seq info [548019:548019:8:4294967295] 00000400:00000200:0.0:1788575457.409349:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575457.409366:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575457.409373:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575457.409380:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575457.409387:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575457.409392:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575457.409411:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575457.409415:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575457.409420:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575457.409425:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575457.409430:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575457.409434:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575457.409440:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575457.409446:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516824832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.409459:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575457.409470:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516824832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.409488:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.409493:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575457.409500:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575457.409506:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575457.409509:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310821101 00000100:00000010:0.0:1788575457.409516:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575457.409523:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575457.409531:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310821101 00000100:00000010:0.0:1788575457.409538:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50900. 00000100:00000001:0.0:1788575457.409544:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575457.409550:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575457.409556:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.409561:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575457.409569:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575457/real 1788575457] req@ffff8fc4901a5880 x1875440516824832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575457.409591:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575457.409593:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575457.409596:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575457.409599:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575457.409602:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575457.409606:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575457.409610:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516824832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575457.409621:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516824832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.409632:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575457.409634:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.409638:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575457.409640:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575457.409643:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516824832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.409653:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575457.409659:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516824832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.409668:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575457.409673:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.409676:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575457.409678:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.409681:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575457.409684:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575457.409687:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575457.409692:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575457.409696:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575457.409702:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516824832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575457.409714:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516824832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575457.409720:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.409725:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575457.409730:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516824832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.409739:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575457.409744:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575457.409747:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575457.409750:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575457.409753:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575457.409755:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575457.409758:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575457.409761:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575457.409765:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575457.409770:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575457.409773:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575457.409778:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575457.409781:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.409785:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.409792:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575457.409796:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.409801:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575457.409805:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575457.409842:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575457.409847:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516824960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.409862:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575457.409864:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575457.409867:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575457.409870:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575457.409873:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575457.409879:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516824960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575457.409883:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575457.409885:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575457.409887:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575457.409890:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575457.409893:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575457.409896:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575457.409903:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575457.409907:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575457.409911:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575457.409915:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516824960, portal 10 00000100:00000001:0.0:1788575457.409918:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575457.409921:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575457.409930:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516824960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.409940:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575457.409944:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575457.409951:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575457.409955:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516824960, offset 0 00000400:00000200:0.0:1788575457.409963:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575457.409968:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575457.409973:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575457.409978:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575457.409982:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575457.409986:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575457.409989:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575457.409994:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548097:548097:256:4294967295] 192.168.202.128@tcp LPNI seq info [548020:548020:8:4294967295] 00000400:00000200:0.0:1788575457.410003:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575457.410014:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575457.410022:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575457.410038:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575457.410047:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575457.410051:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575457.410066:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575457.410069:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575457.410073:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575457.410077:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575457.410082:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575457.410086:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575457.410090:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575457.410094:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516824960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410104:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575457.410109:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516824960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410119:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.410121:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575457.410131:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575457.410136:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575457.410142:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575457.410145:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310817299 00000100:00000001:0.0:1788575457.410149:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310821101 00000100:00000001:0.0:1788575457.410152:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310817299 00000100:00000001:0.0:1788575457.410157:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310821101 00000100:00000001:0.0:1788575457.410161:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575457.410163:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.410165:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575457.410171:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575457/real 1788575457] req@ffff8fc4901a4700 x1875440516824960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575457.410187:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575457.410188:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575457.410190:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575457.410192:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575457.410194:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575457.410196:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575457.410198:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516824960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575457.410204:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516824960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410209:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575457.410211:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410212:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575457.410214:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575457.410217:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516824960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410222:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575457.410226:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516824960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410234:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575457.410236:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410238:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575457.410239:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410241:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575457.410244:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575457.410246:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575457.410248:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575457.410250:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575457.410253:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516824960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575457.410258:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516824960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575457.410260:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575457.410262:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516825088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410266:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575457.410267:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575457.410268:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575457.410269:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575457.410271:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575457.410273:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575457.410275:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575457.410277:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575457.410278:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575457.410279:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575457.410281:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575457.410282:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575457.410287:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575457.410288:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575457.410290:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575457.410292:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516825088, portal 10 00000100:00000001:0.0:1788575457.410293:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575457.410294:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575457.410298:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516825088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410302:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575457.410303:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575457.410305:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575457.410306:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516825088, offset 0 00000400:00000200:0.0:1788575457.410308:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575457.410310:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575457.410313:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575457.410314:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575457.410316:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575457.410317:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575457.410319:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575457.410320:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548098:548098:256:4294967295] 192.168.202.128@tcp LPNI seq info [548021:548021:8:4294967295] 00000400:00000200:0.0:1788575457.410323:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575457.410327:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575457.410329:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575457.410331:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575457.410333:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575457.410335:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575457.410341:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575457.410342:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575457.410344:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575457.410346:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575457.410348:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575457.410351:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575457.410353:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575457.410355:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516825088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410360:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575457.410361:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516825088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410365:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.410366:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575457.410368:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575457.410370:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575457.410372:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575457.410373:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310817306 00000100:00000001:0.0:1788575457.410375:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310821101 00000100:00000001:0.0:1788575457.410376:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310817306 00000100:00000001:0.0:1788575457.410377:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310821101 00000100:00000001:0.0:1788575457.410378:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575457.410379:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.410380:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575457.410382:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575457/real 1788575457] req@ffff8fc4901a7480 x1875440516825088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575457.410386:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575457.410387:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575457.410388:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575457.410389:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575457.410390:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575457.410391:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575457.410393:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516825088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575457.410397:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516825088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410404:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575457.410405:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410406:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575457.410407:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575457.410409:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516825088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410412:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575457.410415:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516825088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410419:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575457.410420:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410421:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575457.410421:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410422:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575457.410424:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575457.410425:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575457.410427:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575457.410428:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575457.410430:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516825088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575457.410434:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575457.410436:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410437:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575457.410438:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516824960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410442:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575457.410445:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575457.410446:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575457.410447:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575457.410449:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575457.410449:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575457.410451:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575457.410452:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575457.410454:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575457.410454:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575457.410456:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575457.410458:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575457.410459:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410460:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575457.410462:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516825088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410465:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575457.410466:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575457.410467:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575457.410469:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575457.410470:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575457.410470:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575457.410471:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575457.410472:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575457.410473:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575457.410473:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575457.410474:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575457.410476:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575457.410477:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410478:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410480:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575457.410481:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.410482:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575457.410483:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575457.410483:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575457.410485:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516825216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410489:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575457.410500:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575457.410502:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575457.410503:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575457.410504:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575457.410506:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575457.410508:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575457.410509:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575457.410510:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575457.410511:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575457.410513:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575457.410514:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575457.410516:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575457.410518:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575457.410520:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575457.410523:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516825216, portal 4 00000100:00000001:0.0:1788575457.410524:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575457.410526:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575457.410531:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516825216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410537:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575457.410539:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575457.410541:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575457.410543:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516825216, offset 0 00000400:00000200:0.0:1788575457.410547:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575457.410550:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575457.410553:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575457.410556:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575457.410559:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575457.410561:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575457.410563:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575457.410566:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548099:548099:256:4294967295] 192.168.202.128@tcp LPNI seq info [548022:548022:8:4294967295] 00000400:00000200:0.0:1788575457.410569:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575457.410574:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575457.410581:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575457.410583:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575457.410585:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575457.410587:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575457.410593:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575457.410594:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575457.410596:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575457.410598:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575457.410600:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575457.410601:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575457.410602:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575457.410605:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516825216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410609:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575457.410611:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516825216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410615:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.410616:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575457.410618:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575457.410619:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575457.410621:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575457.410623:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310817446 00000100:00000001:0.0:1788575457.410624:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310821101 00000100:00000001:0.0:1788575457.410625:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310817446 00000100:00000001:0.0:1788575457.410627:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310821101 00000100:00000001:0.0:1788575457.410628:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575457.410629:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.410630:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575457.410632:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575457/real 1788575457] req@ffff8fc4901a6300 x1875440516825216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575457.410639:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575457.410640:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575457.410641:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575457.410643:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575457.410644:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575457.410645:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575457.410647:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516825216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575457.410651:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516825216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410655:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575457.410656:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410657:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575457.410657:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575457.410659:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516825216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410663:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575457.410665:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516825216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410668:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575457.410669:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410670:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575457.410671:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410672:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575457.410673:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575457.410674:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575457.410676:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575457.410677:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575457.410682:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516825216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575457.410686:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575457.410688:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575457.410690:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516825344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410693:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575457.410694:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575457.410695:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575457.410696:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575457.410697:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575457.410699:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575457.410700:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575457.410701:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575457.410702:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575457.410703:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575457.410705:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575457.410706:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575457.410708:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575457.410710:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575457.410711:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575457.410713:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516825344, portal 4 00000100:00000001:0.0:1788575457.410714:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575457.410715:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575457.410718:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516825344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410722:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575457.410723:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575457.410724:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575457.410726:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516825344, offset 0 00000400:00000200:0.0:1788575457.410731:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575457.410733:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575457.410735:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575457.410737:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575457.410738:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575457.410740:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575457.410741:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575457.410743:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548100:548100:256:4294967295] 192.168.202.128@tcp LPNI seq info [548023:548023:8:4294967295] 00000400:00000200:0.0:1788575457.410745:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575457.410749:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575457.410751:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575457.410752:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575457.410754:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575457.410756:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575457.410762:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575457.410763:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575457.410765:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575457.410766:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575457.410768:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575457.410769:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575457.410770:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575457.410773:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516825344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410777:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575457.410778:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516825344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410782:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.410783:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575457.410785:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575457.410786:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575457.410789:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575457.410795:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310817448 00000100:00000001:0.0:1788575457.410797:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310821101 00000100:00000001:0.0:1788575457.410799:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310817448 00000100:00000001:0.0:1788575457.410801:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310821101 00000100:00000001:0.0:1788575457.410803:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575457.410805:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.410827:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575457.410832:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575457/real 1788575457] req@ffff8fc4901a4e00 x1875440516825344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575457.410838:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575457.410840:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575457.410841:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575457.410843:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575457.410844:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575457.410845:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575457.410848:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516825344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575457.410853:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516825344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410856:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575457.410857:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410858:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575457.410859:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575457.410861:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516825344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410865:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575457.410867:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516825344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410871:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575457.410876:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410877:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575457.410877:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410878:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575457.410880:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575457.410881:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575457.410883:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575457.410884:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575457.410886:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516825344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575457.410891:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575457.410893:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410894:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575457.410895:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516825216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410899:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575457.410899:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575457.410901:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575457.410902:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575457.410903:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575457.410904:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575457.410905:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575457.410906:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575457.410908:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575457.410908:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575457.410909:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575457.410912:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575457.410913:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410914:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575457.410915:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516825344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575457.410926:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575457.410926:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575457.410927:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575457.410929:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575457.410929:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575457.410930:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575457.410931:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575457.410932:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575457.410933:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575457.410933:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575457.410934:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575457.410936:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575457.410936:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410937:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.410939:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575457.410940:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.410941:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575457.410942:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.920912:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575457.920919:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575457.920921:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575457.920926:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575457.920930:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575457.920934:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.920939:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575457.920942:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.920952:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575457.920985:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575457.920997:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575457.921000:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575457.921007:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575457.921013:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575457.921033:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.921039:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575457.921042:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575457.921044:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575458.625069:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575458.625074:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575458.625078:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575458.945001:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575458.945009:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575458.945014:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575458.945024:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575458.945045:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575458.945056:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575458.945064:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575458.945078:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575458.945083:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575459.968914:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575459.968924:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575459.968927:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575459.968935:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575459.968939:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575459.968945:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575459.968953:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575459.968962:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575459.968973:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575460.673058:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575460.673062:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575460.673064:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575460.992998:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575460.993003:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575460.993007:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575460.993011:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575460.993019:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575460.993022:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575460.993037:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575460.993041:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575460.993046:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575461.248948:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310821101, at 4310821120 00000100:00000010:0.0:1788575461.248967:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575461.697044:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575461.697048:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575461.697073:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575462.016949:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575462.016956:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575462.016960:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575462.016971:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575462.016981:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575462.016986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.017001:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575462.017006:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.017011:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.529259:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575462.529266:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575462.529269:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575462.529280:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516825472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.529309:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575462.529312:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.529317:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575462.529322:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575462.529328:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575462.529334:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516825472:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575462.529341:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575462.529346:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575462.529350:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.529354:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575462.529361:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575462.529366:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575462.529377:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575462.529385:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575462.529398:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575462.529411:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516825472, portal 25 00000100:00000001:0.0:1788575462.529417:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575462.529420:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788575462.529432:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516825472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.529463:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575462.529469:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575462.529478:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575462.529485:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516825472, offset 0 00000400:00000200:0.0:1788575462.529496:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575462.529509:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575462.529523:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575462.529534:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575462.529541:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575462.529548:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575462.529556:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575462.529565:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548101:548101:256:4294967295] 192.168.202.128@tcp LPNI seq info [548024:548024:8:4294967295] 00000400:00000200:0.0:1788575462.529576:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575462.529596:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575462.529609:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575462.529620:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575462.529635:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575462.529642:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575462.529675:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575462.529681:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575462.529694:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575462.529702:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575462.529712:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575462.529717:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575462.529733:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575462.529740:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516825472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.529762:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575462.529769:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516825472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.529886:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.529891:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575462.529903:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575462.529914:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575462.529920:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310826222 00000100:00000010:0.0:1788575462.529928:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575462.529940:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575462.529953:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310826222 00000100:00000010:0.0:1788575462.529959:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50960. 00000100:00000001:0.0:1788575462.529966:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575462.529973:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575462.529978:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.529984:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575462.529994:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575462/real 1788575462] req@ffff8fc4ac821c00 x1875440516825472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575462.530014:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575462.530018:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575462.530022:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575462.530027:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575462.530031:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575462.530035:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575462.530042:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516825472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575462.530063:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516825472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530082:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575462.530084:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.530087:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575462.530091:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575462.530096:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516825472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530110:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575462.530115:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516825472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530124:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575462.530128:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.530130:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575462.530131:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.530134:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575462.530147:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575462.530150:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575462.530155:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575462.530159:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575462.530164:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516825472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575462.530173:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516825472:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575462.530180:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.530182:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575462.530186:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516825472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530193:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575462.530196:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575462.530200:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575462.530204:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575462.530206:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575462.530208:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575462.530210:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575462.530214:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788575462.530220:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575462.530226:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575462.530229:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788575462.530233:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575462.530234:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.530237:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.530249:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575462.530252:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.530253:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575462.530255:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575462.530257:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575462.530261:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516825600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530269:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575462.530270:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.530272:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575462.530275:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575462.530277:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575462.530280:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825600:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575462.530284:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575462.530286:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575462.530289:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.530291:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575462.530294:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575462.530296:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575462.530301:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575462.530304:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575462.530309:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575462.530312:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516825600, portal 10 00000100:00000001:0.0:1788575462.530314:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575462.530316:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788575462.530324:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516825600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530332:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575462.530334:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575462.530344:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575462.530347:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516825600, offset 0 00000400:00000200:0.0:1788575462.530352:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575462.530356:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575462.530360:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575462.530364:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575462.530373:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575462.530376:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575462.530379:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575462.530382:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548102:548102:256:4294967295] 192.168.202.128@tcp LPNI seq info [548025:548025:8:4294967295] 00000400:00000200:0.0:1788575462.530386:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575462.530393:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575462.530398:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575462.530402:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575462.530407:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575462.530409:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575462.530421:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575462.530423:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575462.530426:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575462.530429:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575462.530432:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575462.530434:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575462.530437:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575462.530441:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516825600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530449:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575462.530452:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516825600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530458:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.530459:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575462.530471:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575462.530475:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575462.530479:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575462.530481:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310822419 00000100:00000001:0.0:1788575462.530484:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310826222 00000100:00000001:0.0:1788575462.530486:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310822419 00000100:00000001:0.0:1788575462.530488:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310826222 00000100:00000001:0.0:1788575462.530490:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575462.530492:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.530494:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575462.530500:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575462/real 1788575462] req@ffff8fc4ac820a80 x1875440516825600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575462.530509:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575462.530510:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575462.530512:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575462.530515:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575462.530516:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575462.530518:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575462.530522:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516825600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575462.530531:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516825600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530538:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575462.530540:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.530541:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575462.530543:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575462.530548:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516825600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530554:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575462.530558:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516825600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530571:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575462.530574:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.530575:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575462.530577:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.530579:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575462.530581:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575462.530584:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575462.530588:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575462.530592:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575462.530596:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516825600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575462.530605:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825600:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575462.530611:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575462.530615:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516825728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530623:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575462.530625:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.530628:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575462.530630:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575462.530633:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575462.530636:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825728:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575462.530640:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575462.530644:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575462.530647:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.530648:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575462.530653:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575462.530656:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575462.530665:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575462.530669:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575462.530673:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575462.530676:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516825728, portal 10 00000100:00000001:0.0:1788575462.530679:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575462.530680:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788575462.530687:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516825728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530693:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575462.530695:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575462.530697:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575462.530699:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516825728, offset 0 00000400:00000200:0.0:1788575462.530704:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575462.530708:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575462.530712:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575462.530714:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575462.530716:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575462.530719:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575462.530721:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575462.530725:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548103:548103:256:4294967295] 192.168.202.128@tcp LPNI seq info [548026:548026:8:4294967295] 00000400:00000200:0.0:1788575462.530731:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575462.530738:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575462.530743:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575462.530747:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575462.530751:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575462.530754:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575462.530767:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575462.530769:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575462.530772:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575462.530775:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575462.530779:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575462.530787:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575462.530791:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575462.530796:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516825728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530805:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575462.530843:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516825728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530851:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.530854:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575462.530860:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575462.530864:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575462.530869:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575462.530872:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310822427 00000100:00000001:0.0:1788575462.530875:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310826222 00000100:00000001:0.0:1788575462.530878:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310822427 00000100:00000001:0.0:1788575462.530882:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310826222 00000100:00000001:0.0:1788575462.530884:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575462.530886:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.530889:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575462.530893:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575462/real 1788575462] req@ffff8fc4ac823800 x1875440516825728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575462.530902:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575462.530903:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575462.530906:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575462.530908:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575462.530909:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575462.530912:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575462.530915:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516825728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575462.530923:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516825728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530937:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575462.530939:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.530941:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575462.530942:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575462.530946:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516825728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530951:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575462.530954:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516825728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.530958:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575462.530960:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.530962:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575462.530962:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.530964:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575462.530967:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575462.530969:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575462.530972:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575462.530974:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575462.530978:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516825728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575462.530986:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825728:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575462.530991:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575462.530994:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516825856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531001:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575462.531002:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.531010:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575462.531013:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575462.531015:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575462.531019:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825856:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575462.531023:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575462.531025:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575462.531027:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.531029:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575462.531032:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575462.531034:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575462.531039:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575462.531041:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575462.531045:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575462.531048:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516825856, portal 4 00000100:00000001:0.0:1788575462.531050:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575462.531051:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788575462.531057:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516825856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531063:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575462.531066:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575462.531068:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575462.531070:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516825856, offset 0 00000400:00000200:0.0:1788575462.531073:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575462.531077:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575462.531081:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575462.531084:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575462.531086:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575462.531089:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575462.531091:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575462.531093:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548104:548104:256:4294967295] 192.168.202.128@tcp LPNI seq info [548027:548027:8:4294967295] 00000400:00000200:0.0:1788575462.531097:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575462.531104:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575462.531118:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575462.531120:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575462.531123:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575462.531125:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575462.531146:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575462.531149:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575462.531153:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575462.531156:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575462.531159:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575462.531161:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575462.531164:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575462.531167:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516825856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531175:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575462.531178:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516825856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531184:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.531186:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575462.531190:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575462.531193:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575462.531196:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575462.531197:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310822567 00000100:00000001:0.0:1788575462.531199:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310826222 00000100:00000001:0.0:1788575462.531201:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310822567 00000100:00000001:0.0:1788575462.531203:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310826222 00000100:00000001:0.0:1788575462.531204:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575462.531205:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.531206:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575462.531209:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575462/real 1788575462] req@ffff8fc4ac821880 x1875440516825856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575462.531219:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575462.531219:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575462.531221:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575462.531223:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575462.531224:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575462.531225:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575462.531227:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516825856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575462.531232:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516825856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531236:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575462.531237:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.531238:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575462.531238:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575462.531240:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516825856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531244:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575462.531246:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516825856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531251:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575462.531253:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.531254:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575462.531255:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.531257:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575462.531259:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575462.531261:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575462.531264:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575462.531266:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575462.531275:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516825856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575462.531282:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825856:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575462.531286:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575462.531290:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516825984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531297:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575462.531299:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.531301:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575462.531303:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575462.531305:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575462.531308:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825984:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575462.531311:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575462.531313:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575462.531314:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575462.531315:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575462.531318:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575462.531320:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575462.531324:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575462.531326:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575462.531330:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575462.531334:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516825984, portal 4 00000100:00000001:0.0:1788575462.531335:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575462.531337:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788575462.531342:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516825984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531348:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575462.531351:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575462.531353:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575462.531356:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516825984, offset 0 00000400:00000200:0.0:1788575462.531363:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575462.531366:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575462.531368:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575462.531370:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575462.531372:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575462.531374:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575462.531375:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575462.531377:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548105:548105:256:4294967295] 192.168.202.128@tcp LPNI seq info [548028:548028:8:4294967295] 00000400:00000200:0.0:1788575462.531380:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575462.531384:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575462.531387:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575462.531389:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575462.531392:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575462.531393:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575462.531400:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575462.531401:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575462.531403:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575462.531405:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575462.531407:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575462.531409:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575462.531410:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575462.531413:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516825984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531418:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575462.531420:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516825984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531426:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.531428:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575462.531431:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575462.531433:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575462.531437:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575462.531443:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310822569 00000100:00000001:0.0:1788575462.531446:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310826222 00000100:00000001:0.0:1788575462.531448:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310822569 00000100:00000001:0.0:1788575462.531450:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310826222 00000100:00000001:0.0:1788575462.531452:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575462.531454:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.531456:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575462.531460:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575462/real 1788575462] req@ffff8fc4ac820e00 x1875440516825984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575462.531467:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575462.531468:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575462.531470:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575462.531472:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575462.531473:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575462.531476:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575462.531478:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516825984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575462.531485:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516825984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531491:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575462.531493:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.531494:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575462.531496:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575462.531498:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516825984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531505:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575462.531508:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516825984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531515:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575462.531523:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.531524:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575462.531525:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.531527:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575462.531529:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575462.531531:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575462.531535:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575462.531537:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575462.531542:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516825984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575462.531549:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516825984:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575462.531552:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.531555:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575462.531558:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516825600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531564:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575462.531567:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575462.531569:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575462.531572:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575462.531574:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575462.531577:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575462.531579:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575462.531581:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788575462.531585:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575462.531587:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575462.531589:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788575462.531592:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575462.531593:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.531594:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575462.531597:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516825728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531615:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575462.531617:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575462.531618:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575462.531622:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575462.531624:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575462.531625:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575462.531627:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575462.531629:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788575462.531631:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575462.531632:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575462.531633:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788575462.531636:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575462.531637:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.531639:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575462.531642:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516825856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531649:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575462.531650:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575462.531652:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575462.531654:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575462.531655:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575462.531656:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575462.531658:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575462.531659:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788575462.531662:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575462.531663:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575462.531664:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788575462.531667:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575462.531668:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.531669:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575462.531673:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516825984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575462.531679:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575462.531680:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575462.531689:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575462.531692:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575462.531693:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575462.531694:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575462.531696:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575462.531697:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862ac00. 02000000:00000001:0.0:1788575462.531700:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575462.531701:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575462.531702:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788575462.531705:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575462.531706:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.531708:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575462.531716:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575462.531718:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575462.531720:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575462.531721:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575463.040930:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575463.040938:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575463.040939:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575463.040944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575463.040949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575463.040952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575463.040955:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575463.040961:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575463.040967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575463.040986:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575463.040991:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575463.040998:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575463.041009:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575463.041026:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575463.041038:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575463.041045:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575463.041048:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575463.041053:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575463.745054:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575463.745059:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575463.745064:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575464.064871:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575464.064892:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575464.064894:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575464.064901:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575464.064906:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575464.064910:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575464.064916:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575464.064921:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575464.064924:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575464.064962:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575464.064967:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575464.064969:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575464.064971:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575464.064974:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575464.064976:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575464.064977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575464.064978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575464.064980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575465.089010:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575465.089014:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575465.089016:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575465.089018:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575465.089022:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575465.089026:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575465.089029:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575465.089031:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575465.089034:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575465.793052:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575465.793059:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575465.793064:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575466.113015:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575466.113025:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575466.113029:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575466.113040:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575466.113052:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575466.113057:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575466.113061:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575466.113064:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575466.113066:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575466.368887:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310826222, at 4310826240 00000100:00000010:0.0:1788575466.368900:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575466.816975:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575466.816978:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575466.816981:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575467.136951:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575467.136955:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575467.136958:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575467.136963:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575467.136967:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575467.136970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575467.136972:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575467.136973:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575467.136975:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575467.648914:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575467.648923:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575467.648926:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575467.648930:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575467.648934:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575467.648940:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16155 00000400:00000200:0.0:1788575467.648947:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575467.648952:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575467.648954:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575467.648956:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.648959:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575467.648960:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.648962:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575467.648965:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575467.648966:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575467.648968:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575467.648970:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575467.648971:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575467.648973:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575467.648992:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.648993:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575467.648994:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.648996:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575467.648998:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575467.649000:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649002:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575467.649004:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575467.649006:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575467.649008:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649009:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575467.649011:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575467.649017:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575467.649019:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575467.649022:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575467.649023:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649026:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575467.649028:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575467.649032:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575467.649037:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575467.649042:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575467.649048:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516826112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575467.649055:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575467.649058:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575467.649063:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516826112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575467.649083:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575467.649086:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575467.649089:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575467.649091:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575467.649098:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575467.649101:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575467.649108:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16155 00000400:00000200:0.0:1788575467.649116:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575467.649121:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575467.649125:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575467.649131:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649135:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575467.649138:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649142:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575467.649146:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575467.649149:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575467.649151:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575467.649154:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575467.649157:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575467.649160:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575467.649167:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649170:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575467.649174:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649179:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575467.649182:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575467.649186:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649189:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575467.649191:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575467.649197:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575467.649202:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649204:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575467.649207:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575467.649211:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649215:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575467.649222:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575467.649226:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575467.649232:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575467.649234:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649239:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575467.649240:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575467.649243:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575467.649249:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575467.649254:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575467.649262:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516826240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575467.649272:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575467.649276:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575467.649280:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516826240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575467.649308:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575467.649311:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575467.649316:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575467.649317:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575467.649319:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575467.649324:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575467.649329:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16155 00000400:00000200:0.0:1788575467.649338:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575467.649347:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575467.649351:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575467.649359:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649364:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575467.649369:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649372:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575467.649377:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575467.649385:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575467.649387:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575467.649389:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575467.649392:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575467.649397:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575467.649403:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649408:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575467.649412:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649422:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575467.649427:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575467.649432:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649437:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575467.649442:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575467.649444:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575467.649447:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649450:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575467.649453:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575467.649457:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649458:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575467.649461:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575467.649466:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575467.649469:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575467.649472:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649474:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575467.649476:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575467.649478:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575467.649483:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575467.649488:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575467.649495:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516826368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575467.649507:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575467.649509:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575467.649521:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516826368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575467.649530:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575467.649533:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575467.649538:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575467.649539:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575467.649541:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575467.649546:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575467.649550:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16155 00000400:00000200:0.0:1788575467.649556:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575467.649562:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575467.649564:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575467.649569:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649574:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575467.649577:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649581:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575467.649586:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575467.649587:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575467.649588:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575467.649591:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575467.649594:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575467.649598:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575467.649603:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649606:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575467.649611:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649615:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575467.649619:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575467.649625:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649629:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575467.649637:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575467.649640:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575467.649646:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649650:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575467.649654:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575467.649657:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649661:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575467.649665:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575467.649668:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575467.649671:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575467.649673:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649677:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575467.649679:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575467.649682:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575467.649686:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575467.649689:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575467.649698:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516826496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575467.649710:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575467.649713:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575467.649718:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516826496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575467.649739:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575467.649743:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575467.649750:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575467.649751:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575467.649754:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575467.649756:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575467.649760:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16155 00000400:00000200:0.0:1788575467.649766:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575467.649771:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575467.649775:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575467.649781:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649782:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575467.649784:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649785:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575467.649786:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575467.649787:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575467.649791:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575467.649792:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575467.649796:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575467.649798:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575467.649803:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649805:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575467.649839:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649844:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575467.649848:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575467.649850:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575467.649853:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575467.649856:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575467.649859:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575467.649864:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649869:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575467.649873:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575467.649879:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649885:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575467.649892:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575467.649900:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575467.649907:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575467.649910:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575467.649915:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575467.649919:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575467.649923:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575467.649930:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575467.649943:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575467.649952:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516826624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575467.649967:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575467.649971:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575467.649979:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516826624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575467.649996:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575467.650004:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16165) 00000100:00000001:0.0:1788575468.160923:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575468.160929:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575468.160933:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575468.160939:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575468.160943:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575468.160947:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575468.160950:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575468.160953:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575468.160956:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575468.160985:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575468.160988:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575468.160989:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575468.160991:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575468.160992:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575468.160994:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575468.160995:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575468.160996:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575468.160997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575468.865029:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575468.865035:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575468.865038:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575469.184947:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575469.184957:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575469.184962:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575469.184973:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575469.184981:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575469.185004:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575469.185029:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575469.185033:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575469.185038:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575470.208910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575470.208917:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575470.208921:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575470.208927:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575470.208932:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575470.208938:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575470.208951:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575470.208956:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575470.208963:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575470.913038:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575470.913041:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575470.913043:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575471.232915:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575471.232924:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575471.232929:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575471.232936:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575471.232940:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575471.232945:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575471.232950:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575471.232957:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575471.232963:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575471.488882:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310831341, at 4310831360 00000100:00000010:0.0:1788575471.488897:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfae0. 00000020:00000001:0.0:1788575471.936977:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575471.936981:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575471.936984:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575472.256881:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575472.256888:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575472.256891:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575472.256897:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575472.256902:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575472.256912:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575472.256920:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575472.256940:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575472.256946:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575472.257026:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575472.257030:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575472.257032:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575472.257035:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575472.257037:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575472.257039:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575472.257041:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575472.257043:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575472.257045:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575473.280922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575473.280931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575473.280938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575473.280946:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575473.280953:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575473.280960:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575473.280982:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575473.280988:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575473.280994:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575473.985050:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575473.985054:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575473.985058:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575474.304896:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575474.304901:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575474.304904:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575474.304910:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575474.304917:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575474.304922:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575474.304939:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575474.304943:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575474.304947:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575475.328941:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575475.328949:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575475.328954:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575475.328958:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575475.328964:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575475.328970:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575475.329000:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575475.329005:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575475.329010:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575476.033032:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575476.033034:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575476.033036:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575476.352951:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575476.352961:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575476.352964:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575476.352971:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575476.352979:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575476.352989:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575476.353864:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575476.353866:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575476.353869:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575476.608885:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310836461, at 4310836480 00000100:00000010:0.0:1788575476.608900:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf5a0. 00000020:00000001:0.0:1788575477.057029:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575477.057032:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575477.057034:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575477.376996:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575477.377011:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575477.377025:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575477.377041:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575477.377061:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575477.377071:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575477.377119:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575477.377124:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575477.377128:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575478.400948:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575478.400956:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575478.400961:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575478.400968:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575478.400975:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575478.400979:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575478.400997:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575478.401000:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575478.401025:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575479.105146:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575479.105150:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575479.105154:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575479.424954:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575479.424962:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575479.424964:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575479.424974:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575479.424980:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575479.424984:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575479.424991:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575479.424999:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575479.425004:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575479.425051:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575479.425055:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575479.425059:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575479.425063:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575479.425076:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575479.425081:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575479.425086:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575479.425087:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575479.425101:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575480.449009:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575480.449025:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575480.449034:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575480.449053:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575480.449068:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575480.449077:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575480.449088:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575480.449097:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575480.449103:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575481.153075:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575481.153081:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575481.153084:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575481.472960:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575481.472974:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575481.472983:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575481.473003:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575481.473026:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575481.473032:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575481.473051:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575481.473056:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575481.473061:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575481.728900:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310841581, at 4310841600 00000100:00000010:0.0:1788575481.728916:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf6c0. 00000020:00000001:0.0:1788575482.177070:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575482.177073:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575482.177076:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575482.496948:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575482.496958:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575482.496963:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575482.496973:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575482.496978:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575482.496984:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575482.497000:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575482.497004:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575482.497009:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575482.497038:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575482.497042:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575482.497044:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575482.497046:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575482.497049:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575482.497052:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575482.497054:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575482.497056:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575482.497059:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575483.520952:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575483.520962:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575483.520968:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575483.520975:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575483.520979:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575483.520983:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575483.520986:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575483.520989:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575483.520993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575483.521077:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575483.521080:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575483.521082:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575483.521085:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575483.521089:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575483.521093:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575483.521098:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575483.521100:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575483.521104:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575483.521135:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575483.521139:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575483.521140:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575483.521143:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575483.521145:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575483.521148:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575483.521150:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575483.521151:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575483.521154:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575484.225011:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575484.225015:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575484.225018:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575484.544939:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575484.544948:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575484.544953:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575484.544963:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575484.544969:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575484.544974:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575484.544984:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575484.544992:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575484.544997:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575485.569020:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575485.569025:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575485.569027:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575485.569032:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575485.569036:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575485.569039:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575485.569046:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575485.569051:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575485.569065:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575486.273018:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575486.273023:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575486.273026:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575486.592963:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575486.592972:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575486.592975:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575486.592981:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575486.592986:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575486.592990:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575486.592995:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575486.593000:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575486.593007:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575486.593048:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575486.593053:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575486.593055:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575486.593057:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575486.593060:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575486.593063:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575486.593065:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575486.593067:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575486.593069:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575486.848946:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310846701, at 4310846720 00000100:00000010:0.0:1788575486.848965:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfde0. 00000020:00000001:0.0:1788575487.297098:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575487.297102:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575487.297110:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575487.616927:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575487.616933:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575487.616936:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575487.616941:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575487.616945:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575487.616951:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575487.616960:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575487.616965:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575487.616973:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575488.640960:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575488.640983:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575488.640985:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575488.640993:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575488.640996:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575488.640999:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575488.641012:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575488.641016:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575488.641020:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575489.345147:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575489.345152:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575489.345155:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575489.664966:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575489.664979:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575489.664983:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575489.664991:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575489.664999:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575489.665007:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575489.665014:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575489.665019:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575489.665025:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575489.665067:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575489.665071:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575489.665073:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575489.665076:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575489.665079:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575489.665081:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575489.665083:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575489.665085:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575489.665087:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575490.688968:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575490.688975:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575490.688983:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575490.688989:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575490.688998:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575490.689003:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575490.689025:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575490.689029:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575490.689033:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575491.393182:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575491.393185:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575491.393188:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575491.712869:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575491.712881:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575491.712883:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575491.712891:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575491.712906:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575491.712911:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575491.712914:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575491.712918:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575491.712922:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575491.968913:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310851821, at 4310851840 00000100:00000010:0.0:1788575491.968928:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfd80. 00000020:00000001:0.0:1788575492.417050:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575492.417054:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575492.417058:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575492.736916:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575492.736924:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575492.736929:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575492.736936:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575492.736943:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575492.736949:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575492.736963:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575492.736968:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575492.736972:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575492.737011:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575492.737015:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575492.737016:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575492.737019:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575492.737022:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575492.737024:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575492.737026:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575492.737027:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575492.737029:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575493.249105:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788575493.249111:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575493.249128:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575493.249132:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575493.249134:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575493.761007:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575493.761010:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575493.761011:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575493.761013:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575493.761015:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575493.761017:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575493.761018:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575493.761020:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575493.761021:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575493.761131:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575493.761136:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575493.761139:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575493.761143:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575493.761148:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575493.761151:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575493.761156:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575493.761159:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575493.761163:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575494.464982:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575494.464984:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575494.464987:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575495.808949:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575495.808954:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575495.808959:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575495.808967:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575495.808973:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575495.808977:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575495.808991:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575495.808994:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575495.808997:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575496.512946:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575496.512948:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575496.512951:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575496.832901:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575496.832909:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575496.832930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575496.832935:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575496.832940:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575496.832944:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575496.832958:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575496.832963:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575496.832966:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575497.088910:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310856941, at 4310856960 00000100:00000010:0.0:1788575497.088925:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf4e0. 00000020:00000001:0.0:1788575497.536992:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575497.536994:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575497.536997:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575497.856925:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575497.856934:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575497.856938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575497.856948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575497.856956:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575497.856961:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575497.857007:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575497.857012:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575497.857016:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575498.881210:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575498.881214:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575498.881215:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575498.881217:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575498.881220:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575498.881223:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575498.881225:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575498.881226:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575498.881228:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575499.585000:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575499.585004:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575499.585007:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575499.904921:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575499.904928:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575499.904932:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575499.904937:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575499.904961:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575499.904963:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575499.904984:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575499.904988:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575499.904991:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575500.928902:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575500.928908:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575500.928912:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575500.928920:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575500.928927:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575500.928931:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575500.928935:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575500.928939:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575500.928941:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575500.928967:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575500.928971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575500.928974:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575500.928987:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575500.928991:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575500.928993:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575500.928995:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575500.928997:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575500.928999:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575501.632977:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575501.632980:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575501.632983:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575501.952923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575501.952929:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575501.952933:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575501.952938:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575501.952950:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575501.952956:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575501.952970:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575501.952973:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575501.952976:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575501.952993:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575501.952998:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575501.953000:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575501.953025:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575501.953028:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575501.953031:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575501.953032:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575501.953035:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575501.953037:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575502.208920:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310862061, at 4310862080 00000100:00000010:0.0:1788575502.208935:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfa80. 00000020:00000001:0.0:1788575502.656969:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575502.656972:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575502.656975:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575502.976894:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575502.976902:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575502.976906:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575502.976913:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575502.976919:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575502.976924:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575502.976928:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575502.976932:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575502.976937:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575502.976962:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575502.976966:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575502.976968:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575502.976970:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575502.976972:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575502.976974:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575502.976976:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575502.976978:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575502.976980:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575504.000980:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575504.000992:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575504.001001:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575504.001009:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575504.001016:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575504.001021:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575504.001047:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575504.001053:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575504.001065:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575504.705061:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575504.705065:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575504.705069:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575505.024965:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575505.024970:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575505.024974:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575505.024982:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575505.024992:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575505.024996:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575505.025007:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575505.025012:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575505.025019:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575506.048958:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575506.048969:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575506.048973:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575506.048982:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575506.048988:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575506.049005:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575506.049020:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575506.049028:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575506.049033:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575506.752991:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575506.752995:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575506.752998:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575507.072900:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575507.072907:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575507.072912:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575507.072920:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575507.072927:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575507.072932:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575507.072941:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575507.072944:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575507.072948:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575507.328904:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310867181, at 4310867200 00000100:00000010:0.0:1788575507.328921:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf4e0. 00000020:00000001:0.0:1788575507.776977:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575507.776996:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575507.776999:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575508.096927:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575508.096935:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575508.096939:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575508.096944:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575508.096949:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575508.096954:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.098440:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575508.098456:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.098467:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.609107:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575508.609114:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575508.609119:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575508.609130:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516831232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.609148:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575508.609153:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575508.609158:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575508.609163:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575508.609171:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575508.609179:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516831232:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575508.609188:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575508.609193:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575508.609198:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575508.609203:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575508.609210:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575508.609214:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575508.609223:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575508.609231:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575508.609241:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575508.609249:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516831232, portal 25 00000100:00000001:0.0:1788575508.609254:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575508.609257:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575508.609266:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516831232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.609293:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575508.609299:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575508.609303:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575508.609307:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516831232, offset 0 00000400:00000200:0.0:1788575508.609317:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575508.609325:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575508.609333:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575508.609339:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575508.609344:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575508.609349:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575508.609354:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575508.609356:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548146:548146:256:4294967295] 192.168.202.128@tcp LPNI seq info [548069:548069:8:4294967295] 00000400:00000200:0.0:1788575508.609362:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575508.609373:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575508.609382:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575508.609389:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575508.609398:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575508.609405:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575508.609429:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575508.609432:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575508.609440:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575508.609450:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575508.609467:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575508.609472:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575508.609482:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575508.609491:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516831232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.609507:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575508.609514:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516831232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.609549:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.609553:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575508.609562:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575508.609572:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575508.609577:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310872301 00000100:00000010:0.0:1788575508.609585:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575508.609593:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575508.609601:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310872301 00000100:00000010:0.0:1788575508.609608:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b504e0. 00000100:00000001:0.0:1788575508.609612:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575508.609617:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575508.609621:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.609624:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575508.609631:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575508/real 1788575508] req@ffff8fc4901a4e00 x1875440516831232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575508.609655:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575508.609660:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575508.609664:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575508.609671:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575508.609673:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575508.609677:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575508.609684:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516831232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575508.609704:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516831232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.609718:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575508.609720:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.609724:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575508.609726:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575508.609732:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516831232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.609750:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575508.609755:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516831232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.609761:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575508.609764:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.609766:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575508.609768:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.609773:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575508.609775:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575508.609779:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575508.609785:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575508.609790:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575508.609796:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516831232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575508.609832:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516831232:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575508.609837:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.609840:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575508.609845:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516831232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.609858:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575508.609861:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575508.609864:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575508.609869:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575508.609872:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575508.609874:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575508.609878:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575508.609881:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575508.609885:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575508.609892:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575508.609896:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575508.609904:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575508.609908:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.609912:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.609921:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575508.609923:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.609926:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575508.609928:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575508.609930:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575508.609936:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516831360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.609944:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575508.609946:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575508.609949:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575508.609951:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575508.609954:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575508.609957:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516831360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575508.609961:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575508.609963:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575508.609965:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575508.609967:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575508.609970:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575508.609971:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575508.609975:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575508.609978:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575508.609982:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575508.609984:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516831360, portal 10 00000100:00000001:0.0:1788575508.609986:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575508.609987:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575508.609993:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516831360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610000:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575508.610002:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575508.610010:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575508.610013:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516831360, offset 0 00000400:00000200:0.0:1788575508.610018:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575508.610022:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575508.610026:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575508.610029:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575508.610031:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575508.610033:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575508.610035:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575508.610038:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548147:548147:256:4294967295] 192.168.202.128@tcp LPNI seq info [548070:548070:8:4294967295] 00000400:00000200:0.0:1788575508.610042:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575508.610048:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575508.610053:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575508.610056:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575508.610060:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575508.610062:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575508.610073:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575508.610074:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575508.610076:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575508.610078:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575508.610080:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575508.610081:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575508.610083:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575508.610086:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516831360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610091:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575508.610092:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516831360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610096:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.610097:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575508.610104:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575508.610107:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575508.610109:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575508.610111:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310868499 00000100:00000001:0.0:1788575508.610112:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310872301 00000100:00000001:0.0:1788575508.610114:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310868499 00000100:00000001:0.0:1788575508.610115:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310872301 00000100:00000001:0.0:1788575508.610116:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575508.610118:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.610119:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575508.610121:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575508/real 1788575508] req@ffff8fc4901a6300 x1875440516831360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575508.610125:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575508.610126:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575508.610127:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575508.610129:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575508.610129:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575508.610131:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575508.610133:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516831360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575508.610137:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516831360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610141:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575508.610142:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610143:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575508.610143:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575508.610145:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516831360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610149:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575508.610151:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516831360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610157:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575508.610159:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610160:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575508.610161:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610163:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575508.610165:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575508.610167:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575508.610170:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575508.610172:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575508.610176:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516831360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575508.610184:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516831360:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575508.610188:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575508.610192:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516831488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610198:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575508.610199:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575508.610201:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575508.610203:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575508.610204:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575508.610207:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516831488:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575508.610211:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575508.610212:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575508.610214:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575508.610215:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575508.610219:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575508.610220:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575508.610229:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575508.610231:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575508.610235:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575508.610237:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516831488, portal 10 00000100:00000001:0.0:1788575508.610239:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575508.610241:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575508.610247:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516831488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610253:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575508.610255:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575508.610257:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575508.610259:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516831488, offset 0 00000400:00000200:0.0:1788575508.610263:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575508.610266:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575508.610269:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575508.610270:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575508.610272:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575508.610274:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575508.610275:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575508.610277:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548148:548148:256:4294967295] 192.168.202.128@tcp LPNI seq info [548071:548071:8:4294967295] 00000400:00000200:0.0:1788575508.610280:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575508.610288:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575508.610291:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575508.610293:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575508.610295:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575508.610297:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575508.610303:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575508.610304:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575508.610306:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575508.610308:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575508.610310:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575508.610315:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575508.610317:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575508.610319:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516831488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610324:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575508.610326:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516831488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610330:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.610331:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575508.610333:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575508.610335:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575508.610337:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575508.610338:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310868506 00000100:00000001:0.0:1788575508.610339:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310872301 00000100:00000001:0.0:1788575508.610341:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310868506 00000100:00000001:0.0:1788575508.610342:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310872301 00000100:00000001:0.0:1788575508.610343:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575508.610344:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.610345:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575508.610347:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575508/real 1788575508] req@ffff8fc4901a7480 x1875440516831488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575508.610352:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575508.610353:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575508.610354:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575508.610356:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575508.610356:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575508.610358:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575508.610359:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516831488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575508.610364:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516831488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610372:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575508.610373:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610374:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575508.610375:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575508.610377:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516831488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610381:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575508.610383:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516831488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610386:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575508.610388:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610389:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575508.610389:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610390:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575508.610392:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575508.610393:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575508.610395:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575508.610396:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575508.610398:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516831488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575508.610403:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516831488:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575508.610407:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575508.610410:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516831616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610416:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575508.610417:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575508.610423:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575508.610425:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575508.610427:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575508.610430:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516831616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575508.610433:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575508.610435:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575508.610436:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575508.610438:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575508.610442:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575508.610443:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575508.610447:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575508.610449:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575508.610452:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575508.610454:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516831616, portal 4 00000100:00000001:0.0:1788575508.610468:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575508.610469:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575508.610474:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516831616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610478:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575508.610479:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575508.610480:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575508.610482:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516831616, offset 0 00000400:00000200:0.0:1788575508.610484:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575508.610487:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575508.610489:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575508.610491:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575508.610493:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575508.610494:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575508.610495:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575508.610497:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548149:548149:256:4294967295] 192.168.202.128@tcp LPNI seq info [548072:548072:8:4294967295] 00000400:00000200:0.0:1788575508.610499:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575508.610503:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575508.610512:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575508.610515:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575508.610519:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575508.610521:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575508.610531:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575508.610533:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575508.610536:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575508.610539:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575508.610541:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575508.610543:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575508.610546:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575508.610549:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516831616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610556:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575508.610558:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516831616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610562:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.610563:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575508.610565:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575508.610567:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575508.610569:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575508.610571:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310868646 00000100:00000001:0.0:1788575508.610572:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310872301 00000100:00000001:0.0:1788575508.610574:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310868646 00000100:00000001:0.0:1788575508.610575:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310872301 00000100:00000001:0.0:1788575508.610576:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575508.610577:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.610578:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575508.610580:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575508/real 1788575508] req@ffff8fc4901a4700 x1875440516831616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575508.610589:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575508.610590:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575508.610591:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575508.610593:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575508.610593:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575508.610595:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575508.610597:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516831616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575508.610602:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516831616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610605:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575508.610606:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610607:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575508.610608:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575508.610610:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516831616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610613:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575508.610615:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516831616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610618:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575508.610620:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610621:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575508.610622:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610623:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575508.610624:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575508.610626:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575508.610627:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575508.610628:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575508.610633:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516831616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575508.610637:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516831616:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575508.610640:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575508.610642:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516831744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610647:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575508.610648:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575508.610650:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575508.610652:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575508.610654:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575508.610656:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516831744:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575508.610660:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575508.610661:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575508.610663:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575508.610664:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575508.610668:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575508.610670:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575508.610673:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575508.610676:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575508.610679:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575508.610681:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516831744, portal 4 00000100:00000001:0.0:1788575508.610683:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575508.610684:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575508.610689:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516831744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610693:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575508.610695:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575508.610696:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575508.610698:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516831744, offset 0 00000400:00000200:0.0:1788575508.610704:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575508.610706:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575508.610709:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575508.610710:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575508.610712:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575508.610714:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575508.610715:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575508.610717:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548150:548150:256:4294967295] 192.168.202.128@tcp LPNI seq info [548073:548073:8:4294967295] 00000400:00000200:0.0:1788575508.610719:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575508.610723:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575508.610726:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575508.610727:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575508.610730:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575508.610731:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575508.610737:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575508.610738:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575508.610740:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575508.610742:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575508.610744:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575508.610745:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575508.610747:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575508.610748:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516831744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610753:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575508.610755:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516831744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575533 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610759:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.610759:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575508.610761:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575508.610763:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575508.610765:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575508.610769:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310868648 00000100:00000001:0.0:1788575508.610770:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310872301 00000100:00000001:0.0:1788575508.610772:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310868648 00000100:00000001:0.0:1788575508.610773:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310872301 00000100:00000001:0.0:1788575508.610774:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575508.610775:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.610776:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575508.610778:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575508/real 1788575508] req@ffff8fc4901a5880 x1875440516831744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575508.610783:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575508.610784:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575508.610785:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575508.610786:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575508.610787:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575508.610788:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575508.610790:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516831744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575508.610794:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516831744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610798:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575508.610799:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610800:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575508.610800:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575508.610802:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516831744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610806:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575508.610822:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516831744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610828:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575508.610836:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610838:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575508.610839:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610840:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575508.610842:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575508.610844:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575508.610847:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575508.610849:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575508.610853:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516831744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575508.610861:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516831744:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575508.610864:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610866:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575508.610869:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516831360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610875:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575508.610877:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575508.610879:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575508.610881:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575508.610882:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575508.610883:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575508.610885:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575508.610887:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575508.610889:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575508.610890:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575508.610892:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575508.610896:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575508.610897:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610898:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575508.610902:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516831488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610916:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575508.610917:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575508.610918:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575508.610921:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575508.610922:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575508.610922:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575508.610924:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575508.610924:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575508.610926:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575508.610926:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575508.610927:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575508.610929:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575508.610930:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610931:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575508.610933:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516831616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610937:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575508.610937:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575508.610938:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575508.610939:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575508.610940:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575508.610940:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575508.610941:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575508.610942:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575508.610943:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575508.610943:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575508.610944:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575508.610945:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575508.610946:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610947:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575508.610948:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516831744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575533 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575508.610952:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575508.610952:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575508.610955:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575508.610957:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575508.610957:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575508.610958:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575508.610959:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575508.610960:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575508.610962:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575508.610962:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575508.610963:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575508.610965:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575508.610965:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610966:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575508.610969:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575508.610970:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575508.610971:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575508.610972:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575509.121083:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575509.121086:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575509.121090:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575509.121098:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575509.121101:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575509.121104:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575509.121249:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575509.121251:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575509.121254:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575509.825024:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575509.825027:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575509.825029:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575510.144995:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575510.145000:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575510.145003:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575510.145008:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575510.145014:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575510.145018:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575510.145022:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575510.145026:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575510.145030:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575511.168911:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575511.168930:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575511.168935:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575511.168941:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575511.168947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575511.168953:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575511.168959:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575511.168963:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575511.168968:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575511.872987:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575511.872990:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575511.872992:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575512.192921:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575512.192925:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575512.192928:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575512.192935:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575512.192939:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575512.192943:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575512.192948:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575512.192954:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575512.192960:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575512.448963:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310872301, at 4310872320 00000100:00000010:0.0:1788575512.448989:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575512.897067:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575512.897072:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575512.897076:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575513.729099:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575513.729106:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575513.729110:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575513.729120:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516831872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.729139:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575513.729143:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575513.729149:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575513.729160:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575513.729166:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575513.729173:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516831872:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575513.729194:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575513.729198:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575513.729202:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575513.729206:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575513.729212:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575513.729215:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575513.729226:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575513.729231:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575513.729240:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575513.729249:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516831872, portal 25 00000100:00000001:0.0:1788575513.729254:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575513.729258:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575513.729271:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516831872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.729287:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575513.729294:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575513.729298:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575513.729304:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516831872, offset 0 00000400:00000200:0.0:1788575513.729312:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575513.729321:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575513.729330:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575513.729337:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575513.729344:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575513.729350:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575513.729355:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575513.729369:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548151:548151:256:4294967295] 192.168.202.128@tcp LPNI seq info [548074:548074:8:4294967295] 00000400:00000200:0.0:1788575513.729378:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575513.729395:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575513.729404:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575513.729411:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575513.729423:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575513.729434:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575513.729457:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575513.729462:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575513.729469:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575513.729479:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575513.729487:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575513.729492:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575513.729499:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575513.729508:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516831872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.729535:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575513.729541:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516831872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.729559:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575513.729563:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575513.729569:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575513.729577:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575513.729582:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310877421 00000100:00000010:0.0:1788575513.729589:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575513.729596:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575513.729615:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310877421 00000100:00000010:0.0:1788575513.729623:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50ae0. 00000100:00000001:0.0:1788575513.729630:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575513.729635:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575513.729641:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575513.729646:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575513.729655:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575513/real 1788575513] req@ffff8fc4901a7800 x1875440516831872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575513.729679:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575513.729682:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575513.729691:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575513.729695:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575513.729697:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575513.729703:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575513.729710:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516831872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575513.729733:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516831872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.729753:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575513.729757:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.729761:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575513.729765:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575513.729773:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516831872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.729789:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575513.729805:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516831872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.729842:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575513.729849:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.729853:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575513.729856:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.729861:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575513.729864:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575513.729869:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575513.729875:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575513.729877:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575513.729882:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516831872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575513.729891:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516831872:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575513.729907:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.729909:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575513.729913:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516831872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.729919:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575513.729921:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575513.729924:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575513.729927:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575513.729929:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575513.729930:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575513.729932:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575513.729934:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575513.729938:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575513.729940:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575513.729941:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575513.729944:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575513.729945:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.729947:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.729958:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575513.729960:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575513.729962:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575513.729963:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575513.729965:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575513.729968:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516832000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.729976:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575513.729977:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575513.729979:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575513.729981:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575513.729983:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575513.729986:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832000:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575513.729988:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575513.729990:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575513.729992:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575513.730014:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575513.730017:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575513.730019:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575513.730023:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575513.730026:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575513.730033:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575513.730036:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516832000, portal 10 00000100:00000001:0.0:1788575513.730038:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575513.730040:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575513.730047:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516832000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730053:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575513.730055:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575513.730057:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575513.730059:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516832000, offset 0 00000400:00000200:0.0:1788575513.730063:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575513.730067:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575513.730070:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575513.730073:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575513.730076:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575513.730078:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575513.730080:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575513.730083:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548152:548152:256:4294967295] 192.168.202.128@tcp LPNI seq info [548075:548075:8:4294967295] 00000400:00000200:0.0:1788575513.730087:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575513.730094:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575513.730098:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575513.730101:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575513.730106:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575513.730110:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575513.730121:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575513.730123:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575513.730130:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575513.730134:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575513.730137:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575513.730138:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575513.730141:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575513.730144:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516832000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730151:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575513.730154:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516832000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730160:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575513.730161:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575513.730164:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575513.730167:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575513.730170:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575513.730172:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310873619 00000100:00000001:0.0:1788575513.730174:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310877421 00000100:00000001:0.0:1788575513.730176:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310873619 00000100:00000001:0.0:1788575513.730178:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310877421 00000100:00000001:0.0:1788575513.730180:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575513.730182:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575513.730184:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575513.730188:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575513/real 1788575513] req@ffff8fc4901a5c00 x1875440516832000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575513.730195:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575513.730197:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575513.730198:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575513.730200:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575513.730202:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575513.730204:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575513.730215:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516832000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575513.730223:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516832000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730230:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575513.730231:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730233:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575513.730234:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575513.730238:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516832000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730244:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575513.730247:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516832000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730251:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575513.730253:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730254:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575513.730255:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730256:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575513.730257:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575513.730259:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575513.730260:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575513.730262:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575513.730264:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516832000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575513.730268:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832000:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575513.730271:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575513.730273:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516832128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730283:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575513.730284:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575513.730286:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575513.730287:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575513.730289:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575513.730290:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832128:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575513.730293:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575513.730294:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575513.730295:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575513.730296:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575513.730297:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575513.730298:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575513.730301:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575513.730302:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575513.730304:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575513.730306:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516832128, portal 10 00000100:00000001:0.0:1788575513.730308:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575513.730308:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575513.730312:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516832128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730316:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575513.730317:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575513.730318:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575513.730320:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516832128, offset 0 00000400:00000200:0.0:1788575513.730322:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575513.730324:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575513.730326:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575513.730328:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575513.730330:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575513.730331:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575513.730332:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575513.730337:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548153:548153:256:4294967295] 192.168.202.128@tcp LPNI seq info [548076:548076:8:4294967295] 00000400:00000200:0.0:1788575513.730340:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575513.730344:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575513.730347:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575513.730349:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575513.730351:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575513.730353:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575513.730359:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575513.730360:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575513.730362:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575513.730364:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575513.730366:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575513.730367:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575513.730368:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575513.730371:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516832128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730375:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575513.730377:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516832128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730381:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575513.730382:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575513.730384:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575513.730385:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575513.730387:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575513.730388:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310873626 00000100:00000001:0.0:1788575513.730390:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310877421 00000100:00000001:0.0:1788575513.730391:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310873626 00000100:00000001:0.0:1788575513.730392:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310877421 00000100:00000001:0.0:1788575513.730393:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575513.730397:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575513.730399:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575513.730401:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575513/real 1788575513] req@ffff8fc4901a5180 x1875440516832128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575513.730406:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575513.730407:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575513.730408:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575513.730409:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575513.730410:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575513.730411:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575513.730413:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516832128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575513.730418:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516832128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730421:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575513.730422:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730423:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575513.730424:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575513.730426:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516832128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730429:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575513.730431:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516832128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730435:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575513.730436:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730437:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575513.730437:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730438:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575513.730440:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575513.730441:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575513.730446:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575513.730447:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575513.730449:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516832128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575513.730454:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832128:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575513.730456:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575513.730457:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516832256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730461:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575513.730462:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575513.730463:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575513.730464:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575513.730466:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575513.730467:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832256:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575513.730469:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575513.730470:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575513.730470:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575513.730471:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575513.730473:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575513.730474:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575513.730476:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575513.730477:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575513.730479:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575513.730481:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516832256, portal 4 00000100:00000001:0.0:1788575513.730482:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575513.730483:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575513.730486:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516832256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730489:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575513.730493:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575513.730494:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575513.730496:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516832256, offset 0 00000400:00000200:0.0:1788575513.730498:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575513.730501:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575513.730503:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575513.730504:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575513.730506:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575513.730507:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575513.730508:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575513.730510:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548154:548154:256:4294967295] 192.168.202.128@tcp LPNI seq info [548077:548077:8:4294967295] 00000400:00000200:0.0:1788575513.730512:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575513.730516:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575513.730518:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575513.730520:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575513.730522:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575513.730524:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575513.730529:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575513.730531:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575513.730532:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575513.730534:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575513.730536:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575513.730537:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575513.730538:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575513.730540:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516832256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730545:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575513.730546:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516832256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730550:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575513.730551:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575513.730556:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575513.730558:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575513.730559:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575513.730561:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310873766 00000100:00000001:0.0:1788575513.730562:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310877421 00000100:00000001:0.0:1788575513.730563:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310873766 00000100:00000001:0.0:1788575513.730565:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310877421 00000100:00000001:0.0:1788575513.730566:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575513.730567:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575513.730568:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575513.730570:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575513/real 1788575513] req@ffff8fc4901a7100 x1875440516832256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575513.730574:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575513.730575:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575513.730576:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575513.730577:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575513.730578:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575513.730579:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575513.730581:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516832256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575513.730585:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516832256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730588:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575513.730589:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730590:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575513.730590:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575513.730592:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516832256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730596:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575513.730598:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516832256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730621:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575513.730622:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730623:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575513.730624:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730625:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575513.730627:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575513.730628:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575513.730630:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575513.730632:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575513.730634:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516832256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575513.730639:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832256:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575513.730641:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575513.730643:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516832384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730646:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575513.730647:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575513.730648:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575513.730649:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575513.730650:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575513.730652:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832384:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575513.730653:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575513.730654:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575513.730655:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575513.730656:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575513.730658:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575513.730659:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575513.730661:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575513.730666:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575513.730668:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575513.730670:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516832384, portal 4 00000100:00000001:0.0:1788575513.730671:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575513.730672:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575513.730675:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516832384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730679:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575513.730680:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575513.730682:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575513.730683:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516832384, offset 0 00000400:00000200:0.0:1788575513.730685:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575513.730687:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575513.730689:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575513.730691:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575513.730692:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575513.730694:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575513.730695:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575513.730697:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548155:548155:256:4294967295] 192.168.202.128@tcp LPNI seq info [548078:548078:8:4294967295] 00000400:00000200:0.0:1788575513.730699:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575513.730703:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575513.730706:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575513.730707:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575513.730710:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575513.730711:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575513.730717:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575513.730718:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575513.730720:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575513.730722:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575513.730724:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575513.730725:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575513.730730:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575513.730732:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516832384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730736:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575513.730738:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516832384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575538 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730741:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575513.730742:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575513.730744:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575513.730746:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575513.730748:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575513.730749:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310873768 00000100:00000001:0.0:1788575513.730751:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310877421 00000100:00000001:0.0:1788575513.730752:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310873768 00000100:00000001:0.0:1788575513.730753:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310877421 00000100:00000001:0.0:1788575513.730754:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575513.730755:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575513.730756:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575513.730758:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575513/real 1788575513] req@ffff8fc4901a4a80 x1875440516832384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575513.730763:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575513.730764:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575513.730765:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575513.730766:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575513.730767:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575513.730768:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575513.730769:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516832384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575513.730774:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516832384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730783:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575513.730785:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730786:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575513.730786:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575513.730789:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516832384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730793:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575513.730795:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516832384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730798:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575513.730799:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730800:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575513.730801:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730802:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575513.730804:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575513.730805:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575513.730829:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575513.730831:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575513.730835:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516832384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575513.730842:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832384:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575513.730846:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730848:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575513.730851:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516832000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730856:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575513.730858:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575513.730863:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575513.730865:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575513.730866:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575513.730867:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575513.730868:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575513.730869:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575513.730871:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575513.730872:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575513.730873:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575513.730876:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575513.730877:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730878:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575513.730880:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516832128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730885:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575513.730885:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575513.730887:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575513.730888:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575513.730889:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575513.730890:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575513.730891:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575513.730891:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575513.730892:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575513.730893:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575513.730893:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575513.730895:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575513.730895:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730896:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575513.730898:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516832256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730901:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575513.730901:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575513.730902:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575513.730903:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575513.730904:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575513.730904:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575513.730908:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575513.730909:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575513.730910:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575513.730911:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575513.730911:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575513.730913:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575513.730913:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730914:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575513.730916:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516832384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575538 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575513.730919:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575513.730920:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575513.730921:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575513.730922:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575513.730922:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575513.730923:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575513.730924:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575513.730924:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575513.730926:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575513.730926:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575513.730927:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575513.730929:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575513.730930:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730931:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575513.730933:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575513.730934:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575513.730935:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575513.730936:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575514.241026:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575514.241037:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575514.241043:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575514.241054:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575514.241065:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575514.241076:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575514.241084:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575514.241091:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575514.241096:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575514.945027:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575514.945029:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575514.945031:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575515.264942:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575515.264958:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575515.264965:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575515.264980:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575515.264989:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575515.265001:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575515.265679:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575515.265682:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575515.265684:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575516.288931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575516.288939:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575516.288943:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575516.288950:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575516.288957:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575516.288963:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575516.288974:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575516.288979:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575516.288983:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575516.993045:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575516.993048:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575516.993051:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575517.312980:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575517.312990:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575517.312995:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575517.313003:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575517.313014:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575517.313023:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575517.313038:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575517.313042:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575517.313294:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575517.313362:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575517.313366:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575517.313368:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575517.313370:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575517.313373:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575517.313397:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575517.313399:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575517.313400:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575517.313402:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575517.568953:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310877421, at 4310877440 00000100:00000010:0.0:1788575517.568977:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575518.016992:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575518.016994:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575518.016997:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575518.336979:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575518.336987:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575518.336991:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575518.336996:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575518.336999:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575518.337003:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.337006:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575518.337009:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.337012:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.849141:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575518.849146:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575518.849157:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575518.849168:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516832512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.849180:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575518.849182:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575518.849187:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575518.849189:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575518.849193:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575518.849196:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516832512:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575518.849201:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575518.849205:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575518.849208:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575518.849211:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575518.849215:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575518.849218:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575518.849237:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575518.849243:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575518.849250:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575518.849256:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516832512, portal 25 00000100:00000001:0.0:1788575518.849260:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575518.849262:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575518.849271:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516832512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.849280:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575518.849286:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575518.849289:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575518.849293:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516832512, offset 0 00000400:00000200:0.0:1788575518.849302:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575518.849310:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575518.849317:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575518.849322:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575518.849328:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575518.849334:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575518.849337:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575518.849344:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548156:548156:256:4294967295] 192.168.202.128@tcp LPNI seq info [548079:548079:8:4294967295] 00000400:00000200:0.0:1788575518.849350:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575518.849366:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575518.849378:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575518.849387:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575518.849397:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575518.849404:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575518.849433:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575518.849437:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575518.849444:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575518.849454:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575518.849461:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575518.849470:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575518.849477:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575518.849485:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516832512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.849501:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575518.849508:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516832512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.849518:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.849521:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575518.849528:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575518.849536:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575518.849540:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310882541 00000100:00000010:0.0:1788575518.849545:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575518.849553:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575518.849559:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310882541 00000100:00000010:0.0:1788575518.849565:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b507e0. 00000100:00000001:0.0:1788575518.849569:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575518.849574:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575518.849578:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.849582:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575518.849588:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575518/real 1788575518] req@ffff8fc4901a4a80 x1875440516832512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575518.849605:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575518.849608:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575518.849611:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575518.849614:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575518.849617:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575518.849622:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575518.849626:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516832512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575518.849642:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516832512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.849664:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575518.849666:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.849670:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575518.849673:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575518.849682:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516832512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.849702:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575518.849713:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516832512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.849731:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575518.849737:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.849741:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575518.849744:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.849777:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575518.849781:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575518.849792:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575518.849802:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575518.849806:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575518.849844:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516832512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575518.849861:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516832512:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575518.849872:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.849877:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575518.849885:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516832512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.849913:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575518.849918:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575518.849927:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575518.849931:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575518.849933:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575518.849935:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575518.849938:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575518.849941:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575518.849945:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575518.849947:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575518.849949:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575518.849953:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575518.849955:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.849958:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.849965:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575518.849967:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.849970:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575518.849972:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575518.849973:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575518.849978:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516832640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.849986:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575518.849987:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575518.849989:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575518.849991:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575518.849993:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575518.849996:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832640:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575518.849999:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575518.850002:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575518.850003:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575518.850005:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575518.850008:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575518.850010:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575518.850014:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575518.850016:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575518.850020:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575518.850023:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516832640, portal 10 00000100:00000001:0.0:1788575518.850029:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575518.850031:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575518.850038:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516832640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850045:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575518.850048:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575518.850050:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575518.850052:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516832640, offset 0 00000400:00000200:0.0:1788575518.850056:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575518.850060:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575518.850064:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575518.850066:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575518.850069:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575518.850071:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575518.850074:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575518.850077:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548157:548157:256:4294967295] 192.168.202.128@tcp LPNI seq info [548080:548080:8:4294967295] 00000400:00000200:0.0:1788575518.850080:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575518.850086:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575518.850091:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575518.850094:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575518.850097:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575518.850100:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575518.850111:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575518.850113:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575518.850116:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575518.850119:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575518.850122:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575518.850124:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575518.850126:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575518.850129:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516832640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850148:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575518.850152:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516832640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850158:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.850160:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575518.850164:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575518.850167:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575518.850170:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575518.850172:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310878739 00000100:00000001:0.0:1788575518.850175:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310882541 00000100:00000001:0.0:1788575518.850177:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310878739 00000100:00000001:0.0:1788575518.850179:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310882541 00000100:00000001:0.0:1788575518.850181:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575518.850183:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.850184:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575518.850188:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575518/real 1788575518] req@ffff8fc4901a7100 x1875440516832640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575518.850195:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575518.850197:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575518.850198:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575518.850200:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575518.850202:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575518.850204:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575518.850207:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516832640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575518.850215:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516832640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850221:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575518.850231:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.850233:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575518.850234:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575518.850237:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516832640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850243:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575518.850247:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516832640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850253:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575518.850256:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.850257:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575518.850259:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.850260:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575518.850262:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575518.850264:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575518.850267:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575518.850270:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575518.850273:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516832640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575518.850281:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832640:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575518.850284:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575518.850287:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516832768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850293:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575518.850294:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575518.850296:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575518.850298:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575518.850300:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575518.850308:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832768:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575518.850311:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575518.850313:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575518.850314:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575518.850316:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575518.850319:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575518.850321:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575518.850324:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575518.850327:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575518.850329:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575518.850332:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516832768, portal 10 00000100:00000001:0.0:1788575518.850334:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575518.850335:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575518.850340:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516832768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850345:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575518.850347:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575518.850349:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575518.850351:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516832768, offset 0 00000400:00000200:0.0:1788575518.850354:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575518.850358:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575518.850361:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575518.850364:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575518.850366:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575518.850368:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575518.850370:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575518.850373:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548158:548158:256:4294967295] 192.168.202.128@tcp LPNI seq info [548081:548081:8:4294967295] 00000400:00000200:0.0:1788575518.850377:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575518.850382:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575518.850387:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575518.850390:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575518.850399:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575518.850402:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575518.850416:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575518.850417:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575518.850420:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575518.850423:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575518.850427:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575518.850428:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575518.850431:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575518.850434:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516832768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850443:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575518.850445:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516832768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850451:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.850453:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575518.850455:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575518.850458:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575518.850461:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575518.850463:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310878746 00000100:00000001:0.0:1788575518.850465:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310882541 00000100:00000001:0.0:1788575518.850467:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310878746 00000100:00000001:0.0:1788575518.850469:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310882541 00000100:00000001:0.0:1788575518.850471:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575518.850473:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.850475:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575518.850478:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575518/real 1788575518] req@ffff8fc4901a5180 x1875440516832768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575518.850486:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575518.850491:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575518.850493:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575518.850495:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575518.850496:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575518.850499:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575518.850502:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516832768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575518.850510:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516832768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850515:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575518.850517:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.850519:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575518.850520:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575518.850524:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516832768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850530:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575518.850533:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516832768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850540:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575518.850542:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.850543:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575518.850544:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.850546:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575518.850548:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575518.850550:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575518.850553:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575518.850555:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575518.850558:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516832768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575518.850575:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832768:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575518.850579:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575518.850582:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516832896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850589:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575518.850590:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575518.850592:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575518.850594:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575518.850595:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575518.850598:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832896:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575518.850601:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575518.850602:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575518.850604:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575518.850605:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575518.850608:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575518.850610:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575518.850614:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575518.850616:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575518.850619:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575518.850622:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516832896, portal 4 00000100:00000001:0.0:1788575518.850623:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575518.850625:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575518.850630:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516832896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850636:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575518.850638:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575518.850640:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575518.850642:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516832896, offset 0 00000400:00000200:0.0:1788575518.850645:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575518.850653:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575518.850657:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575518.850659:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575518.850661:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575518.850663:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575518.850665:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575518.850668:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548159:548159:256:4294967295] 192.168.202.128@tcp LPNI seq info [548082:548082:8:4294967295] 00000400:00000200:0.0:1788575518.850672:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575518.850677:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575518.850681:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575518.850684:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575518.850687:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575518.850690:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575518.850699:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575518.850701:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575518.850704:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575518.850707:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575518.850710:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575518.850711:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575518.850714:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575518.850717:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516832896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850723:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575518.850726:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516832896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850731:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.850732:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575518.850735:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575518.850738:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575518.850741:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575518.850763:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310878886 00000100:00000001:0.0:1788575518.850766:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310882541 00000100:00000001:0.0:1788575518.850768:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310878886 00000100:00000001:0.0:1788575518.850770:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310882541 00000100:00000001:0.0:1788575518.850772:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575518.850774:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.850776:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575518.850780:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575518/real 1788575518] req@ffff8fc4901a5c00 x1875440516832896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575518.850787:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575518.850789:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575518.850790:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575518.850793:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575518.850794:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575518.850797:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575518.850800:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516832896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575518.850824:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516832896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850831:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575518.850833:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.850834:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575518.850835:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575518.850839:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516832896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850844:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575518.850848:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516832896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850854:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575518.850861:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.850863:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575518.850864:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.850866:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575518.850868:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575518.850871:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575518.850874:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575518.850876:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575518.850880:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516832896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575518.850886:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516832896:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575518.850890:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575518.850893:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516833024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850898:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575518.850900:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575518.850902:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575518.850904:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575518.850906:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575518.850909:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516833024:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575518.850912:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575518.850913:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575518.850915:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575518.850917:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575518.850920:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575518.850921:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575518.850924:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575518.850927:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575518.850930:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575518.850932:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516833024, portal 4 00000100:00000001:0.0:1788575518.850938:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575518.850940:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575518.850945:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516833024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.850950:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575518.850952:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575518.850955:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575518.850957:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516833024, offset 0 00000400:00000200:0.0:1788575518.850961:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575518.850965:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575518.850968:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575518.850971:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575518.850973:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575518.850975:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575518.850977:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575518.850979:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548160:548160:256:4294967295] 192.168.202.128@tcp LPNI seq info [548083:548083:8:4294967295] 00000400:00000200:0.0:1788575518.850983:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575518.850989:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575518.850992:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575518.850996:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575518.850999:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575518.851002:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575518.851012:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575518.851014:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575518.851016:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575518.851019:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575518.851022:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575518.851024:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575518.851026:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575518.851029:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516833024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.851046:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575518.851049:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516833024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575543 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.851055:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.851056:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575518.851060:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575518.851062:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575518.851065:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575518.851068:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310878889 00000100:00000001:0.0:1788575518.851070:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310882541 00000100:00000001:0.0:1788575518.851072:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310878889 00000100:00000001:0.0:1788575518.851074:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310882541 00000100:00000001:0.0:1788575518.851076:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575518.851078:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.851079:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575518.851083:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575518/real 1788575518] req@ffff8fc4901a7800 x1875440516833024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575518.851090:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575518.851091:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575518.851092:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575518.851095:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575518.851096:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575518.851099:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575518.851101:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516833024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575518.851109:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516833024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.851116:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575518.851122:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.851123:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575518.851125:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575518.851128:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516833024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.851134:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575518.851138:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516833024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.851145:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575518.851147:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.851149:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575518.851150:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.851152:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575518.851154:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575518.851156:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575518.851159:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575518.851161:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575518.851164:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516833024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575518.851171:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516833024:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575518.851175:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.851177:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575518.851180:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516832640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.851188:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575518.851189:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575518.851191:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575518.851194:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575518.851195:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575518.851201:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575518.851203:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575518.851205:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575518.851208:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575518.851209:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575518.851211:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575518.851214:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575518.851216:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.851217:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575518.851221:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516832768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.851227:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575518.851229:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575518.851230:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575518.851233:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575518.851234:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575518.851236:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575518.851237:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575518.851238:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575518.851240:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575518.851241:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575518.851243:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575518.851246:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575518.851248:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.851249:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575518.851252:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516832896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.851258:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575518.851259:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575518.851261:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575518.851263:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575518.851264:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575518.851265:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575518.851267:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575518.851268:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575518.851270:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575518.851276:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575518.851278:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575518.851280:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575518.851282:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.851283:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575518.851286:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516833024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575543 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575518.851292:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575518.851293:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575518.851295:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575518.851297:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575518.851298:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575518.851300:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575518.851301:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575518.851303:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575518.851306:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575518.851307:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575518.851308:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575518.851311:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575518.851312:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.851314:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575518.851317:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575518.851318:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575518.851320:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575518.851321:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575519.360939:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575519.360947:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575519.360954:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575519.360962:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575519.360975:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575519.360987:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575519.360997:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575519.361005:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575519.361012:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575520.065007:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575520.065011:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575520.065015:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575520.384953:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575520.384965:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575520.384970:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575520.384982:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575520.384989:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575520.384995:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575520.385016:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575520.385021:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575520.385024:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575521.408972:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575521.408981:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575521.408985:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575521.408998:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575521.409011:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575521.409015:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575521.409044:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575521.409047:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575521.409052:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575522.113122:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575522.113127:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575522.113131:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575522.432950:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575522.432958:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575522.432964:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575522.432970:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575522.432974:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575522.432979:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575522.432996:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575522.432999:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575522.433003:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575522.688914:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310882541, at 4310882560 00000100:00000010:0.0:1788575522.688930:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575523.137025:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575523.137030:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575523.137034:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575523.457021:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575523.457033:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575523.457051:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575523.457056:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575523.457060:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575523.457063:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.457070:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575523.457073:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.457075:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.969158:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575523.969164:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575523.969168:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575523.969178:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516833152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.969188:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575523.969191:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575523.969196:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575523.969200:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575523.969204:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575523.969211:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516833152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575523.969217:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575523.969222:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575523.969226:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575523.969230:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575523.969237:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575523.969243:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575523.969252:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575523.969259:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575523.969269:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575523.969279:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516833152, portal 25 00000100:00000001:0.0:1788575523.969303:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575523.969308:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575523.969321:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516833152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.969344:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575523.969351:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575523.969381:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575523.969389:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516833152, offset 0 00000400:00000200:0.0:1788575523.969402:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575523.969415:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575523.969428:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575523.969440:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575523.969448:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575523.969460:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575523.969465:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575523.969472:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548161:548161:256:4294967295] 192.168.202.128@tcp LPNI seq info [548084:548084:8:4294967295] 00000400:00000200:0.0:1788575523.969479:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575523.969494:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575523.969503:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575523.969509:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575523.969517:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575523.969522:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575523.969549:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575523.969552:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575523.969560:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575523.969567:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575523.969575:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575523.969580:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575523.969588:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575523.969596:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516833152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.969617:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575523.969628:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516833152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.969645:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.969649:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575523.969662:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575523.969671:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575523.969674:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310887661 00000100:00000010:0.0:1788575523.969681:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575523.969690:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575523.969698:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310887661 00000100:00000010:0.0:1788575523.969703:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788575523.969709:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575523.969713:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575523.969718:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.969723:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575523.969731:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575523/real 1788575523] req@ffff8fc4901a5880 x1875440516833152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575523.969741:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575523.969743:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575523.969744:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575523.969747:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575523.969748:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575523.969751:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575523.969754:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516833152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575523.969762:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516833152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.969767:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575523.969769:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.969771:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575523.969773:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575523.969776:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516833152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.969782:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575523.969786:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516833152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.969795:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575523.969798:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.969799:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575523.969800:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.969802:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575523.969803:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575523.969806:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575523.969827:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575523.969829:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575523.969832:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516833152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575523.969838:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516833152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575523.969842:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.969843:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575523.969845:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516833152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.969848:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575523.969851:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575523.969853:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575523.969856:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575523.969857:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575523.969858:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575523.969860:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575523.969862:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575523.969865:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575523.969867:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575523.969868:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575523.969872:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575523.969873:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.969875:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.969895:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575523.969896:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.969897:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575523.969899:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575523.969900:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575523.969903:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516833280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.969907:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575523.969908:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575523.969910:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575523.969911:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575523.969912:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575523.969914:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516833280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575523.969916:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575523.969917:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575523.969918:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575523.969919:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575523.969921:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575523.969923:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575523.969930:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575523.969932:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575523.969934:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575523.969936:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516833280, portal 10 00000100:00000001:0.0:1788575523.969938:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575523.969939:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575523.969944:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516833280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.969948:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575523.969949:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575523.969951:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575523.969952:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516833280, offset 0 00000400:00000200:0.0:1788575523.969954:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575523.969957:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575523.969962:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575523.969965:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575523.969966:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575523.969968:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575523.969970:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575523.969972:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548162:548162:256:4294967295] 192.168.202.128@tcp LPNI seq info [548085:548085:8:4294967295] 00000400:00000200:0.0:1788575523.969975:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575523.969979:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575523.969982:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575523.969984:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575523.969986:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575523.969988:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575523.969995:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575523.969996:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575523.969998:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575523.970000:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575523.970002:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575523.970004:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575523.970005:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575523.970008:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516833280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970012:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575523.970014:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516833280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970018:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.970019:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575523.970021:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575523.970024:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575523.970026:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575523.970027:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310883859 00000100:00000001:0.0:1788575523.970034:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310887661 00000100:00000001:0.0:1788575523.970036:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310883859 00000100:00000001:0.0:1788575523.970037:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310887661 00000100:00000001:0.0:1788575523.970039:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575523.970040:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.970041:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575523.970043:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575523/real 1788575523] req@ffff8fc4901a4700 x1875440516833280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575523.970048:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575523.970048:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575523.970050:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575523.970051:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575523.970052:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575523.970053:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575523.970055:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516833280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575523.970059:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516833280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970062:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575523.970063:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970064:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575523.970065:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575523.970067:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516833280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970070:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575523.970073:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516833280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970077:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575523.970078:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970082:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575523.970083:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970084:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575523.970085:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575523.970087:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575523.970089:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575523.970090:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575523.970092:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516833280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575523.970097:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516833280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575523.970100:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575523.970102:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516833408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970106:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575523.970106:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575523.970107:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575523.970108:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575523.970110:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575523.970111:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516833408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575523.970113:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575523.970114:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575523.970115:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575523.970115:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575523.970118:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575523.970119:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575523.970121:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575523.970123:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575523.970125:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575523.970126:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516833408, portal 10 00000100:00000001:0.0:1788575523.970128:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575523.970143:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575523.970148:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516833408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970152:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575523.970153:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575523.970155:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575523.970156:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516833408, offset 0 00000400:00000200:0.0:1788575523.970158:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575523.970161:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575523.970163:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575523.970164:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575523.970166:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575523.970167:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575523.970169:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575523.970171:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548163:548163:256:4294967295] 192.168.202.128@tcp LPNI seq info [548086:548086:8:4294967295] 00000400:00000200:0.0:1788575523.970173:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575523.970177:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575523.970180:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575523.970182:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575523.970184:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575523.970186:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575523.970192:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575523.970193:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575523.970195:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575523.970197:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575523.970199:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575523.970200:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575523.970201:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575523.970203:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516833408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970214:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575523.970216:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516833408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970220:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.970221:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575523.970224:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575523.970226:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575523.970228:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575523.970229:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310883866 00000100:00000001:0.0:1788575523.970231:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310887661 00000100:00000001:0.0:1788575523.970232:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310883866 00000100:00000001:0.0:1788575523.970234:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310887661 00000100:00000001:0.0:1788575523.970235:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575523.970236:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.970237:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575523.970239:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575523/real 1788575523] req@ffff8fc4901a7480 x1875440516833408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575523.970243:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575523.970244:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575523.970245:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575523.970246:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575523.970246:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575523.970247:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575523.970249:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516833408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575523.970253:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516833408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970257:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575523.970261:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970262:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575523.970263:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575523.970265:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516833408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970269:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575523.970271:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516833408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970274:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575523.970276:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970277:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575523.970278:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970279:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575523.970281:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575523.970283:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575523.970284:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575523.970285:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575523.970287:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516833408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575523.970291:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516833408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575523.970293:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575523.970295:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516833536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970299:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575523.970300:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575523.970301:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575523.970302:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575523.970303:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575523.970308:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516833536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575523.970310:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575523.970311:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575523.970312:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575523.970313:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575523.970315:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575523.970316:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575523.970318:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575523.970320:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575523.970322:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575523.970324:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516833536, portal 4 00000100:00000001:0.0:1788575523.970325:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575523.970326:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575523.970329:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516833536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970333:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575523.970335:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575523.970336:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575523.970337:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516833536, offset 0 00000400:00000200:0.0:1788575523.970339:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575523.970341:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575523.970343:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575523.970344:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575523.970346:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575523.970347:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575523.970348:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575523.970350:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548164:548164:256:4294967295] 192.168.202.128@tcp LPNI seq info [548087:548087:8:4294967295] 00000400:00000200:0.0:1788575523.970353:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575523.970357:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575523.970359:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575523.970361:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575523.970367:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575523.970369:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575523.970375:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575523.970376:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575523.970378:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575523.970380:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575523.970382:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575523.970383:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575523.970385:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575523.970387:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516833536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970392:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575523.970393:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516833536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970397:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.970398:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575523.970399:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575523.970401:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575523.970403:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575523.970404:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310884006 00000100:00000001:0.0:1788575523.970406:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310887661 00000100:00000001:0.0:1788575523.970407:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310884006 00000100:00000001:0.0:1788575523.970408:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310887661 00000100:00000001:0.0:1788575523.970410:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575523.970411:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.970412:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575523.970414:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575523/real 1788575523] req@ffff8fc4901a6300 x1875440516833536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575523.970418:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575523.970421:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575523.970422:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575523.970424:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575523.970425:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575523.970426:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575523.970428:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516833536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575523.970433:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516833536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970437:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575523.970438:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970439:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575523.970439:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575523.970441:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516833536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970445:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575523.970446:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516833536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970450:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575523.970451:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970452:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575523.970453:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970454:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575523.970455:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575523.970456:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575523.970458:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575523.970460:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575523.970462:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516833536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575523.970472:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516833536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575523.970480:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575523.970482:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516833664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970486:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575523.970487:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575523.970488:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575523.970489:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575523.970490:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575523.970492:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516833664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575523.970494:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575523.970495:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575523.970496:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575523.970497:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575523.970499:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575523.970500:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575523.970502:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575523.970504:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575523.970506:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575523.970507:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516833664, portal 4 00000100:00000001:0.0:1788575523.970509:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575523.970510:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575523.970513:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516833664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970517:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575523.970519:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575523.970520:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575523.970521:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516833664, offset 0 00000400:00000200:0.0:1788575523.970523:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575523.970529:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575523.970531:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575523.970533:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575523.970535:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575523.970536:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575523.970538:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575523.970540:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548165:548165:256:4294967295] 192.168.202.128@tcp LPNI seq info [548088:548088:8:4294967295] 00000400:00000200:0.0:1788575523.970542:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575523.970546:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575523.970548:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575523.970549:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575523.970552:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575523.970553:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575523.970560:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575523.970561:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575523.970563:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575523.970565:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575523.970567:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575523.970568:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575523.970569:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575523.970572:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516833664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970576:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575523.970578:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516833664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575548 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970582:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.970583:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575523.970584:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575523.970586:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575523.970588:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575523.970592:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310884008 00000100:00000001:0.0:1788575523.970594:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310887661 00000100:00000001:0.0:1788575523.970595:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310884008 00000100:00000001:0.0:1788575523.970596:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310887661 00000100:00000001:0.0:1788575523.970598:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575523.970599:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.970600:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575523.970603:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575523/real 1788575523] req@ffff8fc4901a4e00 x1875440516833664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575523.970607:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575523.970607:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575523.970608:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575523.970609:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575523.970610:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575523.970611:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575523.970613:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516833664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575523.970618:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516833664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970621:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575523.970622:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970623:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575523.970624:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575523.970626:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516833664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970629:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575523.970631:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516833664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970634:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575523.970638:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970639:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575523.970640:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970641:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575523.970643:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575523.970644:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575523.970646:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575523.970647:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575523.970649:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516833664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575523.970653:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516833664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575523.970656:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970657:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575523.970659:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516833280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970662:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575523.970664:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575523.970665:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575523.970667:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575523.970668:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575523.970670:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575523.970671:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575523.970673:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575523.970676:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575523.970677:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575523.970678:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575523.970682:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575523.970683:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970685:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575523.970688:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516833408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970705:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575523.970707:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575523.970708:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575523.970711:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575523.970712:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575523.970713:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575523.970716:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575523.970717:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575523.970719:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575523.970720:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575523.970721:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575523.970724:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575523.970725:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970726:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575523.970729:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516833536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970733:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575523.970734:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575523.970735:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575523.970736:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575523.970737:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575523.970737:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575523.970738:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575523.970739:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575523.970740:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575523.970740:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575523.970741:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575523.970745:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575523.970746:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970746:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575523.970748:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516833664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575548 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575523.970751:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575523.970752:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575523.970757:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575523.970758:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575523.970759:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575523.970760:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575523.970762:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575523.970763:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575523.970764:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575523.970764:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575523.970765:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575523.970767:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575523.970768:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970769:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575523.970771:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575523.970772:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575523.970773:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575523.970774:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575524.480931:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575524.480941:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575524.480946:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575524.480953:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575524.480961:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575524.480968:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575524.480989:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575524.480992:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575524.480995:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575525.185009:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575525.185012:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575525.185015:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575525.504971:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575525.504976:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575525.504982:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575525.504986:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575525.504991:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575525.504995:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575525.505003:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575525.505007:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575525.505010:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575526.528915:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575526.528953:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575526.528957:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575526.528963:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575526.528968:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575526.528973:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575526.528992:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575526.528995:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575526.528999:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575527.233031:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575527.233034:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575527.233037:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575527.552967:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575527.552978:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575527.552987:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575527.553003:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575527.553012:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575527.553018:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575527.553044:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575527.553049:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575527.553056:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575527.553113:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575527.553118:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575527.553121:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575527.553124:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575527.553128:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575527.553131:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575527.553134:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575527.553136:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575527.553139:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575527.808908:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310887661, at 4310887680 00000100:00000010:0.0:1788575527.808921:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575528.256979:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575528.256983:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575528.256985:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575528.576988:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575528.576995:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575528.576997:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575528.577020:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575528.577026:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575528.577031:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575528.577041:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575528.577046:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575528.577051:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.089234:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575529.089243:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575529.089246:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575529.089261:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516833792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.089283:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575529.089289:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575529.089302:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575529.089311:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575529.089322:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575529.089333:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516833792:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575529.089344:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575529.089351:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575529.089356:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575529.089364:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575529.089372:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575529.089386:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575529.089396:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575529.089404:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575529.089411:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575529.089419:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516833792, portal 25 00000100:00000001:0.0:1788575529.089424:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575529.089427:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788575529.089434:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516833792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.089443:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575529.089456:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575529.089463:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575529.089482:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516833792, offset 0 00000400:00000200:0.0:1788575529.089497:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575529.089510:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575529.089524:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575529.089533:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575529.089541:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575529.089545:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575529.089551:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575529.089556:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548166:548166:256:4294967295] 192.168.202.128@tcp LPNI seq info [548089:548089:8:4294967295] 00000400:00000200:0.0:1788575529.089569:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575529.089590:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575529.089602:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575529.089612:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575529.089638:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575529.089649:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788575529.089692:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575529.089698:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575529.089713:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575529.089729:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575529.089744:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575529.089750:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575529.089760:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575529.089771:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516833792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.089843:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575529.089852:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516833792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.089874:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.089883:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575529.089900:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575529.089915:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575529.089937:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310892782 00000100:00000010:0.0:1788575529.089948:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575529.089971:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575529.089981:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310892782 00000100:00000010:0.0:1788575529.089992:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b506c0. 00000100:00000001:0.0:1788575529.090000:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575529.090005:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575529.090011:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.090018:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575529.090033:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575529/real 1788575529] req@ffff8fc4ac823b80 x1875440516833792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575529.090072:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575529.090087:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575529.090094:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575529.090104:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575529.090112:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575529.090119:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575529.090132:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516833792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575529.090210:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516833792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.090263:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575529.090267:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.090282:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575529.090296:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575529.090318:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516833792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.090338:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575529.090352:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516833792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.090435:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575529.090448:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.090455:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575529.090465:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.090474:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575529.090480:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575529.090488:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575529.090506:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575529.090518:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575529.090523:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516833792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575529.090545:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516833792:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575529.090564:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.090571:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575529.090578:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516833792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.090595:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575529.090600:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575529.090607:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575529.090618:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575529.090624:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575529.090630:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575529.090639:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575529.090646:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788575529.090652:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575529.090656:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575529.090661:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788575529.090666:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575529.090670:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.090678:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.090695:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575529.090698:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.090701:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575529.090704:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575529.090706:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575529.090714:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516833920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.090721:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575529.090723:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575529.090725:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575529.090726:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575529.090728:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575529.090730:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516833920:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575529.090734:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575529.090736:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575529.090737:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575529.090739:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575529.090742:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575529.090743:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575529.090748:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575529.090750:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575529.090754:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575529.090758:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516833920, portal 10 00000100:00000001:0.0:1788575529.090760:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575529.090761:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788575529.090768:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516833920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.090773:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575529.090775:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575529.090777:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575529.090779:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516833920, offset 0 00000400:00000200:0.0:1788575529.090788:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575529.090792:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575529.090800:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575529.090803:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575529.090806:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575529.090837:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575529.090839:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575529.090842:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548167:548167:256:4294967295] 192.168.202.128@tcp LPNI seq info [548090:548090:8:4294967295] 00000400:00000200:0.0:1788575529.090845:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575529.090851:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575529.090855:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575529.090858:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575529.090863:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575529.090865:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788575529.090878:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575529.090880:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575529.090883:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575529.090887:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575529.090892:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575529.090894:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575529.090896:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575529.090901:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516833920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.090908:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575529.090911:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516833920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.090918:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.090919:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575529.090923:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575529.090928:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575529.090931:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575529.090933:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310888980 00000100:00000001:0.0:1788575529.090943:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310892782 00000100:00000001:0.0:1788575529.090945:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310888980 00000100:00000001:0.0:1788575529.090947:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310892782 00000100:00000001:0.0:1788575529.090950:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575529.090951:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.090953:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575529.090957:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575529/real 1788575529] req@ffff8fc4ac821500 x1875440516833920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575529.090977:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575529.090979:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575529.090981:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575529.090983:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575529.090985:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575529.090988:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575529.090991:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516833920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575529.090999:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516833920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091005:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575529.091007:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091008:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575529.091010:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575529.091013:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516833920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091020:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575529.091024:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516833920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091031:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575529.091032:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091038:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575529.091039:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091040:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575529.091042:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575529.091043:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575529.091046:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575529.091047:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575529.091050:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516833920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575529.091055:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516833920:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575529.091057:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575529.091060:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516834048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091064:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575529.091065:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575529.091067:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575529.091068:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575529.091070:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575529.091071:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834048:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575529.091073:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575529.091075:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575529.091076:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575529.091077:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575529.091079:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575529.091080:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575529.091082:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575529.091084:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575529.091086:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575529.091088:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516834048, portal 10 00000100:00000001:0.0:1788575529.091089:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575529.091092:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788575529.091097:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516834048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091101:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575529.091103:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575529.091104:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575529.091105:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516834048, offset 0 00000400:00000200:0.0:1788575529.091108:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575529.091110:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575529.091112:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575529.091114:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575529.091115:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575529.091117:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575529.091118:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575529.091120:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548168:548168:256:4294967295] 192.168.202.128@tcp LPNI seq info [548091:548091:8:4294967295] 00000400:00000200:0.0:1788575529.091123:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575529.091127:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575529.091129:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575529.091131:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575529.091135:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575529.091137:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788575529.091147:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575529.091148:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575529.091151:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575529.091154:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575529.091157:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575529.091158:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575529.091161:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575529.091164:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516834048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091182:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575529.091185:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516834048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091191:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.091192:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575529.091196:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575529.091199:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575529.091202:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575529.091203:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310888987 00000100:00000001:0.0:1788575529.091206:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310892782 00000100:00000001:0.0:1788575529.091208:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310888987 00000100:00000001:0.0:1788575529.091210:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310892782 00000100:00000001:0.0:1788575529.091212:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575529.091214:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.091215:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575529.091218:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575529/real 1788575529] req@ffff8fc4ac822a00 x1875440516834048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575529.091223:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575529.091224:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575529.091225:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575529.091226:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575529.091227:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575529.091229:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575529.091231:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516834048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575529.091236:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516834048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091240:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575529.091244:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091246:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575529.091247:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575529.091249:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516834048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091253:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575529.091256:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516834048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091260:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575529.091261:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091262:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575529.091262:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091263:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575529.091265:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575529.091266:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575529.091268:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575529.091269:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575529.091271:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516834048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575529.091276:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834048:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575529.091278:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575529.091280:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516834176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091284:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575529.091285:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575529.091287:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575529.091288:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575529.091289:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575529.091294:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834176:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575529.091296:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575529.091297:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575529.091299:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575529.091300:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575529.091302:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575529.091303:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575529.091305:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575529.091307:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575529.091309:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575529.091311:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516834176, portal 4 00000100:00000001:0.0:1788575529.091312:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575529.091313:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788575529.091317:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516834176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091321:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575529.091322:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575529.091323:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575529.091324:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516834176, offset 0 00000400:00000200:0.0:1788575529.091326:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575529.091329:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575529.091331:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575529.091333:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575529.091334:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575529.091336:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575529.091337:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575529.091339:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548169:548169:256:4294967295] 192.168.202.128@tcp LPNI seq info [548092:548092:8:4294967295] 00000400:00000200:0.0:1788575529.091342:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575529.091346:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575529.091349:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575529.091351:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575529.091359:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575529.091361:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788575529.091372:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575529.091374:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575529.091377:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575529.091381:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575529.091384:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575529.091386:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575529.091389:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575529.091393:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516834176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091401:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575529.091404:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516834176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091410:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.091411:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575529.091414:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575529.091416:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575529.091418:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575529.091420:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310889127 00000100:00000001:0.0:1788575529.091422:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310892782 00000100:00000001:0.0:1788575529.091423:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310889127 00000100:00000001:0.0:1788575529.091425:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310892782 00000100:00000001:0.0:1788575529.091426:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575529.091428:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.091429:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575529.091433:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575529/real 1788575529] req@ffff8fc4ac823100 x1875440516834176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575529.091437:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575529.091443:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575529.091444:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575529.091446:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575529.091447:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575529.091449:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575529.091451:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516834176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575529.091455:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516834176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091459:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575529.091460:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091461:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575529.091462:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575529.091464:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516834176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091468:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575529.091470:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516834176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091474:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575529.091475:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091476:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575529.091477:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091478:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575529.091479:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575529.091481:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575529.091483:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575529.091484:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575529.091486:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516834176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575529.091497:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834176:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575529.091500:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575529.091502:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516834304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091509:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575529.091511:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575529.091513:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575529.091515:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575529.091517:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575529.091521:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834304:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575529.091524:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575529.091527:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575529.091528:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575529.091530:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575529.091533:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575529.091535:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575529.091539:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575529.091541:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575529.091544:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575529.091547:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516834304, portal 4 00000100:00000001:0.0:1788575529.091549:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575529.091550:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788575529.091555:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516834304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091561:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575529.091563:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575529.091565:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575529.091566:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516834304, offset 0 00000400:00000200:0.0:1788575529.091570:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575529.091585:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575529.091589:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575529.091591:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575529.091593:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575529.091596:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575529.091598:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575529.091600:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548170:548170:256:4294967295] 192.168.202.128@tcp LPNI seq info [548093:548093:8:4294967295] 00000400:00000200:0.0:1788575529.091604:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575529.091610:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575529.091614:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575529.091617:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575529.091621:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575529.091624:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788575529.091638:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575529.091640:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575529.091642:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575529.091645:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575529.091648:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575529.091650:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575529.091652:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575529.091656:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516834304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091663:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575529.091666:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516834304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575554 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091672:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.091673:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575529.091676:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575529.091679:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575529.091682:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575529.091688:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310889129 00000100:00000001:0.0:1788575529.091690:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310892782 00000100:00000001:0.0:1788575529.091692:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310889129 00000100:00000001:0.0:1788575529.091694:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310892782 00000100:00000001:0.0:1788575529.091696:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575529.091697:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.091699:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575529.091703:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575529/real 1788575529] req@ffff8fc4ac820000 x1875440516834304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575529.091710:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575529.091712:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575529.091713:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575529.091715:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575529.091717:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575529.091719:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575529.091722:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516834304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575529.091731:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516834304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091737:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575529.091738:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091740:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575529.091741:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575529.091744:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516834304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091750:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575529.091754:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516834304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091760:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575529.091766:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091768:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575529.091769:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091770:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575529.091772:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575529.091774:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575529.091778:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575529.091780:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575529.091783:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516834304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575529.091790:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834304:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575529.091794:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091795:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575529.091798:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516833920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091804:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575529.091805:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575529.091826:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575529.091830:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575529.091832:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575529.091834:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575529.091836:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575529.091838:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788575529.091841:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575529.091843:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575529.091845:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788575529.091848:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575529.091849:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091851:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575529.091855:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516834048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091871:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575529.091873:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575529.091875:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575529.091877:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575529.091878:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575529.091879:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575529.091881:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575529.091883:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788575529.091885:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575529.091886:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575529.091887:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788575529.091890:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575529.091891:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091893:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575529.091896:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516834176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091902:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575529.091904:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575529.091905:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575529.091907:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575529.091908:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575529.091909:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575529.091911:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575529.091912:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788575529.091914:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575529.091915:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575529.091916:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788575529.091918:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575529.091919:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091921:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575529.091924:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516834304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575554 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575529.091929:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575529.091931:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575529.091938:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575529.091940:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575529.091942:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575529.091943:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575529.091945:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575529.091946:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788575529.091949:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575529.091950:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575529.091951:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788575529.091954:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575529.091955:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091957:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.091974:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575529.091976:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.091978:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575529.091980:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.600919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575529.600927:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575529.600930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575529.600937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575529.600944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575529.600951:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.600963:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575529.600967:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.600981:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575529.601024:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575529.601028:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575529.601030:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575529.601033:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575529.601037:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575529.601039:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.601042:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575529.601044:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575529.601046:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575530.305025:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575530.305028:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575530.305031:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575530.624912:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575530.624938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575530.624944:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575530.624947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575530.624953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575530.624958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575530.624970:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575530.624977:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575530.624998:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575531.648900:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575531.648907:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575531.648911:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575531.648918:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575531.648925:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575531.648930:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575531.648938:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575531.648941:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575531.648946:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575531.649012:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575531.649019:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575531.649022:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575531.649033:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575531.649041:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575531.649045:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575531.649071:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575531.649074:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575531.649077:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575532.161257:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788575532.353029:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575532.353034:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575532.353037:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575532.672886:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575532.672896:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575532.672898:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575532.672903:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575532.672909:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575532.672914:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575532.672917:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575532.672920:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575532.672937:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575532.928893:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310892782, at 4310892800 00000100:00000010:0.0:1788575532.928906:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575533.376958:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575533.376962:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575533.376965:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575533.696943:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575533.696948:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575533.696951:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575533.696954:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575533.696958:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575533.696961:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575533.696971:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575533.696973:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575533.696976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.209131:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575534.209137:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575534.209142:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575534.209152:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516834432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.209175:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575534.209178:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575534.209185:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575534.209190:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575534.209195:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575534.209202:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516834432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575534.209208:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575534.209213:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575534.209216:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575534.209221:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575534.209227:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575534.209231:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575534.209240:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575534.209246:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575534.209256:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575534.209276:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516834432, portal 25 00000100:00000001:0.0:1788575534.209282:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575534.209285:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575534.209295:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516834432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.209314:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575534.209320:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575534.209325:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575534.209330:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516834432, offset 0 00000400:00000200:0.0:1788575534.209342:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575534.209352:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575534.209361:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575534.209367:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575534.209375:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575534.209382:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575534.209387:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575534.209394:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548171:548171:256:4294967295] 192.168.202.128@tcp LPNI seq info [548094:548094:8:4294967295] 00000400:00000200:0.0:1788575534.209403:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575534.209421:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575534.209431:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575534.209438:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575534.209447:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575534.209453:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788575534.209477:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575534.209480:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575534.209489:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575534.209499:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575534.209505:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575534.209510:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575534.209518:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575534.209525:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516834432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.209547:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575534.209555:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516834432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.209570:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.209575:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575534.209582:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575534.209588:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575534.209592:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310897901 00000100:00000010:0.0:1788575534.209598:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575534.209607:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575534.209613:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310897901 00000100:00000010:0.0:1788575534.209618:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50a20. 00000100:00000001:0.0:1788575534.209624:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575534.209629:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575534.209636:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.209641:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575534.209649:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575534/real 1788575534] req@ffff8fc4901a4e00 x1875440516834432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575534.209671:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575534.209675:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575534.209680:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575534.209685:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575534.209689:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575534.209695:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575534.209703:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516834432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575534.209722:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516834432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.209741:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575534.209748:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.209753:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575534.209756:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575534.209765:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516834432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.209782:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575534.209789:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516834432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.209798:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575534.209801:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.209803:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575534.209805:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.209841:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575534.209844:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575534.209847:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575534.209851:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575534.209854:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575534.209860:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516834432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575534.209870:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516834432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575534.209875:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.209877:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575534.209880:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516834432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.209886:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575534.209888:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575534.209891:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575534.209893:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575534.209895:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575534.209900:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575534.209902:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575534.209904:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575534.209908:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575534.209910:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575534.209911:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575534.209917:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575534.209918:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.209920:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.209926:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575534.209927:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.209929:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575534.209931:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575534.209932:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575534.209936:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516834560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.209943:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575534.209945:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575534.209947:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575534.209948:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575534.209950:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575534.209953:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575534.209956:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575534.209958:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575534.209959:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575534.209961:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575534.209964:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575534.209965:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575534.209969:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575534.209972:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575534.209975:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575534.209977:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516834560, portal 10 00000100:00000001:0.0:1788575534.209979:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575534.209981:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575534.209986:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516834560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.209998:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575534.210001:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575534.210003:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575534.210005:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516834560, offset 0 00000400:00000200:0.0:1788575534.210009:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575534.210012:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575534.210016:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575534.210019:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575534.210021:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575534.210023:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575534.210025:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575534.210029:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548172:548172:256:4294967295] 192.168.202.128@tcp LPNI seq info [548095:548095:8:4294967295] 00000400:00000200:0.0:1788575534.210032:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575534.210038:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575534.210058:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575534.210061:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575534.210065:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575534.210068:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788575534.210078:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575534.210079:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575534.210082:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575534.210085:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575534.210088:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575534.210090:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575534.210092:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575534.210096:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516834560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210104:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575534.210107:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516834560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210123:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.210124:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575534.210128:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575534.210131:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575534.210135:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575534.210137:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310894099 00000100:00000001:0.0:1788575534.210140:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310897901 00000100:00000001:0.0:1788575534.210142:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310894099 00000100:00000001:0.0:1788575534.210144:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310897901 00000100:00000001:0.0:1788575534.210146:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575534.210148:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.210150:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575534.210154:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575534/real 1788575534] req@ffff8fc4901a6300 x1875440516834560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575534.210162:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575534.210163:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575534.210165:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575534.210167:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575534.210169:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575534.210171:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575534.210174:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516834560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575534.210182:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516834560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210189:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575534.210190:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.210192:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575534.210193:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575534.210196:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516834560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210207:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575534.210211:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516834560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210217:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575534.210219:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.210221:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575534.210222:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.210224:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575534.210226:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575534.210228:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575534.210232:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575534.210234:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575534.210238:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516834560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575534.210246:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575534.210249:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575534.210252:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516834688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210259:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575534.210260:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575534.210262:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575534.210263:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575534.210265:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575534.210268:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575534.210271:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575534.210273:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575534.210277:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575534.210279:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575534.210281:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575534.210283:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575534.210286:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575534.210288:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575534.210291:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575534.210294:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516834688, portal 10 00000100:00000001:0.0:1788575534.210295:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575534.210297:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575534.210302:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516834688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210309:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575534.210311:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575534.210313:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575534.210315:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516834688, offset 0 00000400:00000200:0.0:1788575534.210319:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575534.210322:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575534.210325:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575534.210327:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575534.210330:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575534.210332:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575534.210334:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575534.210337:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548173:548173:256:4294967295] 192.168.202.128@tcp LPNI seq info [548096:548096:8:4294967295] 00000400:00000200:0.0:1788575534.210341:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575534.210347:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575534.210352:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575534.210355:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575534.210358:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575534.210361:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788575534.210371:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575534.210376:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575534.210379:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575534.210382:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575534.210385:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575534.210387:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575534.210389:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575534.210393:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516834688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210400:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575534.210403:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516834688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210410:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.210411:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575534.210414:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575534.210416:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575534.210420:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575534.210421:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310894106 00000100:00000001:0.0:1788575534.210424:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310897901 00000100:00000001:0.0:1788575534.210426:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310894106 00000100:00000001:0.0:1788575534.210428:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310897901 00000100:00000001:0.0:1788575534.210430:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575534.210432:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.210434:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575534.210437:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575534/real 1788575534] req@ffff8fc4901a7480 x1875440516834688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575534.210444:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575534.210445:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575534.210447:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575534.210449:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575534.210451:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575534.210456:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575534.210459:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516834688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575534.210467:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516834688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210473:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575534.210475:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.210476:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575534.210477:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575534.210481:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516834688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210488:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575534.210492:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516834688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210499:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575534.210501:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.210502:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575534.210503:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.210505:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575534.210507:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575534.210509:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575534.210512:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575534.210514:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575534.210518:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516834688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575534.210525:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575534.210528:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575534.210534:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516834816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210540:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575534.210542:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575534.210544:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575534.210546:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575534.210547:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575534.210550:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575534.210553:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575534.210554:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575534.210556:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575534.210558:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575534.210560:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575534.210562:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575534.210565:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575534.210568:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575534.210570:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575534.210573:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516834816, portal 4 00000100:00000001:0.0:1788575534.210575:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575534.210576:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575534.210581:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516834816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210587:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575534.210589:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575534.210591:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575534.210593:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516834816, offset 0 00000400:00000200:0.0:1788575534.210597:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575534.210600:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575534.210603:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575534.210605:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575534.210608:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575534.210610:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575534.210612:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575534.210619:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548174:548174:256:4294967295] 192.168.202.128@tcp LPNI seq info [548097:548097:8:4294967295] 00000400:00000200:0.0:1788575534.210623:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575534.210629:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575534.210634:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575534.210636:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575534.210640:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575534.210642:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788575534.210652:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575534.210654:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575534.210656:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575534.210659:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575534.210662:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575534.210663:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575534.210666:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575534.210669:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516834816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210676:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575534.210679:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516834816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210686:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.210687:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575534.210689:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575534.210692:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575534.210694:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575534.210696:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310894246 00000100:00000001:0.0:1788575534.210698:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310897901 00000100:00000001:0.0:1788575534.210700:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310894246 00000100:00000001:0.0:1788575534.210702:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310897901 00000100:00000001:0.0:1788575534.210704:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575534.210709:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.210711:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575534.210714:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575534/real 1788575534] req@ffff8fc4901a4700 x1875440516834816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575534.210722:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575534.210724:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575534.210725:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575534.210727:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575534.210729:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575534.210731:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575534.210734:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516834816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575534.210742:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516834816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210748:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575534.210749:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.210751:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575534.210752:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575534.210756:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516834816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210762:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575534.210766:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516834816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210772:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575534.210775:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.210776:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575534.210777:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.210779:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575534.210780:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575534.210783:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575534.210788:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575534.210790:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575534.210794:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516834816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575534.210802:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575534.210805:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575534.210827:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516834944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210835:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575534.210837:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575534.210839:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575534.210841:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575534.210843:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575534.210846:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575534.210849:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575534.210851:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575534.210852:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575534.210854:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575534.210857:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575534.210859:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575534.210862:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575534.210864:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575534.210867:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575534.210870:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516834944, portal 4 00000100:00000001:0.0:1788575534.210872:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575534.210873:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575534.210878:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516834944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210884:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575534.210890:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575534.210892:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575534.210894:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516834944, offset 0 00000400:00000200:0.0:1788575534.210898:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575534.210901:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575534.210904:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575534.210907:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575534.210909:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575534.210911:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575534.210914:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575534.210917:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548175:548175:256:4294967295] 192.168.202.128@tcp LPNI seq info [548098:548098:8:4294967295] 00000400:00000200:0.0:1788575534.210921:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575534.210927:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575534.210931:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575534.210933:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575534.210937:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575534.210939:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788575534.210950:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575534.210951:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575534.210954:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575534.210957:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575534.210960:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575534.210962:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575534.210964:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575534.210967:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516834944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210975:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575534.210978:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516834944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575559 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.210984:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.210985:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575534.210993:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575534.210996:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575534.210999:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575534.211001:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310894249 00000100:00000001:0.0:1788575534.211003:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310897901 00000100:00000001:0.0:1788575534.211005:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310894249 00000100:00000001:0.0:1788575534.211007:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310897901 00000100:00000001:0.0:1788575534.211009:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575534.211011:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.211013:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575534.211016:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575534/real 1788575534] req@ffff8fc4901a5880 x1875440516834944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575534.211024:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575534.211025:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575534.211027:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575534.211029:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575534.211030:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575534.211032:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575534.211035:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516834944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575534.211054:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516834944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.211059:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575534.211061:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.211063:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575534.211064:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575534.211068:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516834944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.211075:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575534.211078:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516834944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.211089:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575534.211091:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.211093:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575534.211094:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.211095:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575534.211097:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575534.211099:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575534.211102:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575534.211104:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575534.211108:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516834944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575534.211116:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516834944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575534.211119:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.211121:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575534.211124:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516834560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.211129:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575534.211130:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575534.211132:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575534.211135:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575534.211136:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575534.211137:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575534.211139:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575534.211141:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575534.211144:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575534.211145:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575534.211146:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575534.211151:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575534.211152:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.211157:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575534.211161:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516834688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.211167:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575534.211168:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575534.211170:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575534.211172:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575534.211173:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575534.211174:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575534.211176:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575534.211177:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575534.211179:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575534.211180:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575534.211181:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575534.211184:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575534.211185:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.211186:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575534.211189:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516834816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.211196:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575534.211197:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575534.211198:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575534.211201:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575534.211202:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575534.211203:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575534.211204:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575534.211206:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575534.211207:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575534.211208:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575534.211210:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575534.211212:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575534.211213:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.211214:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575534.211217:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516834944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575559 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575534.211233:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575534.211235:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575534.211236:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575534.211238:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575534.211239:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575534.211240:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575534.211242:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575534.211243:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575534.211245:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575534.211246:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575534.211247:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575534.211250:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575534.211251:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.211252:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.211255:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575534.211257:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.211259:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575534.211260:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575534.720917:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575534.720924:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575534.720929:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575534.720937:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575534.720943:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575534.720947:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.720963:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575534.720965:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575534.720967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575535.424927:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575535.424929:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575535.424932:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575535.744922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575535.744930:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575535.744933:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575535.744940:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575535.744947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575535.744952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575535.744984:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575535.744987:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575535.744989:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575536.768904:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575536.768911:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575536.768916:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575536.768924:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575536.768931:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575536.768936:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575536.768953:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575536.768955:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575536.768958:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575537.472987:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575537.472989:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575537.472991:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575537.792925:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575537.792934:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575537.792938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575537.792945:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575537.792951:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575537.792957:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575537.792974:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575537.792978:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575537.792982:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575538.048930:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310897901, at 4310897920 00000100:00000010:0.0:1788575538.048944:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575538.497011:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575538.497014:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575538.497016:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575538.816916:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575538.816923:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575538.816928:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575538.816936:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575538.816941:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575538.816946:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575538.817249:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575538.817252:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575538.817263:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.329104:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575539.329109:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575539.329114:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575539.329134:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820000 x1875440516835072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.329152:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575539.329155:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575539.329161:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575539.329165:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575539.329168:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575539.329172:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516835072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575539.329176:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575539.329181:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575539.329184:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575539.329186:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575539.329193:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575539.329197:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575539.329205:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575539.329208:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575539.329214:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575539.329223:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516835072, portal 25 00000100:00000001:0.0:1788575539.329228:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575539.329230:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603403776 : -123400106147840 : ffff8fc4ac820000) 00000100:00000040:0.0:1788575539.329236:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820000 x1875440516835072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.329247:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575539.329253:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575539.329256:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575539.329259:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516835072, offset 0 00000400:00000200:0.0:1788575539.329267:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575539.329275:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575539.329283:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575539.329297:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575539.329302:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575539.329308:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575539.329310:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575539.329314:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548176:548176:256:4294967295] 192.168.202.128@tcp LPNI seq info [548099:548099:8:4294967295] 00000400:00000200:0.0:1788575539.329322:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575539.329334:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575539.329341:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575539.329347:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575539.329355:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575539.329359:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1788575539.329371:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575539.329372:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575539.329379:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575539.329385:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575539.329389:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575539.329390:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575539.329394:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575539.329399:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820000 x1875440516835072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.329421:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575539.329429:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820000 x1875440516835072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.329447:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.329450:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575539.329456:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575539.329465:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575539.329468:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310903021 00000100:00000010:0.0:1788575539.329473:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575539.329480:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575539.329493:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310903021 00000100:00000010:0.0:1788575539.329498:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50420. 00000100:00000001:0.0:1788575539.329502:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575539.329506:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575539.329510:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.329515:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575539.329523:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575539/real 1788575539] req@ffff8fc4ac820000 x1875440516835072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575539.329535:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575539.329539:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575539.329542:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575539.329546:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575539.329548:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575539.329551:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575539.329556:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820000 x1875440516835072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575539.329565:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820000 x1875440516835072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.329573:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575539.329576:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.329579:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575539.329581:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575539.329592:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820000 x1875440516835072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.329617:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575539.329628:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820000 x1875440516835072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.329653:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575539.329655:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.329656:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575539.329662:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.329665:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575539.329668:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575539.329671:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575539.329676:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575539.329679:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575539.329685:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820000 x1875440516835072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575539.329700:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516835072:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575539.329706:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.329710:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575539.329715:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820000 x1875440516835072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.329728:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575539.329730:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575539.329732:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575539.329735:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575539.329737:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575539.329739:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575539.329743:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575539.329747:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788575539.329752:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575539.329755:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575539.329758:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820000. 00000100:00000001:0.0:1788575539.329763:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575539.329767:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.329771:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.329781:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575539.329785:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.329790:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575539.329794:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575539.329799:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575539.329807:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823100 x1875440516835200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.329866:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575539.329868:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575539.329870:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575539.329872:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575539.329874:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575539.329878:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516835200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575539.329881:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575539.329883:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575539.329884:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575539.329886:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575539.329889:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575539.329891:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575539.329896:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575539.329898:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575539.329902:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575539.329905:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516835200, portal 10 00000100:00000001:0.0:1788575539.329906:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575539.329908:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603416320 : -123400106135296 : ffff8fc4ac823100) 00000100:00000040:0.0:1788575539.329914:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823100 x1875440516835200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.329920:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575539.329922:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575539.329924:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575539.329926:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516835200, offset 0 00000400:00000200:0.0:1788575539.329930:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575539.329934:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575539.329937:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575539.329940:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575539.329943:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575539.329945:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575539.329947:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575539.329954:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548177:548177:256:4294967295] 192.168.202.128@tcp LPNI seq info [548100:548100:8:4294967295] 00000400:00000200:0.0:1788575539.329958:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575539.329964:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575539.329968:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575539.329972:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575539.329975:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575539.329978:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1788575539.329988:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575539.329990:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575539.329993:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575539.329996:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575539.329999:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575539.330000:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575539.330003:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575539.330006:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823100 x1875440516835200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330013:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575539.330015:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823100 x1875440516835200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330021:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.330022:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575539.330025:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575539.330028:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575539.330031:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575539.330033:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310899219 00000100:00000001:0.0:1788575539.330036:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310903021 00000100:00000001:0.0:1788575539.330038:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310899219 00000100:00000001:0.0:1788575539.330040:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310903021 00000100:00000001:0.0:1788575539.330042:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575539.330048:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.330050:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575539.330054:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575539/real 1788575539] req@ffff8fc4ac823100 x1875440516835200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575539.330063:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575539.330064:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575539.330065:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575539.330068:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575539.330069:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575539.330071:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575539.330074:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823100 x1875440516835200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575539.330083:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823100 x1875440516835200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330089:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575539.330090:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330092:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575539.330093:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575539.330097:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823100 x1875440516835200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330103:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575539.330107:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823100 x1875440516835200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330114:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575539.330117:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330118:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575539.330131:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330133:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575539.330135:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575539.330138:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575539.330146:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575539.330149:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575539.330153:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823100 x1875440516835200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575539.330160:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516835200:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575539.330164:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575539.330167:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822a00 x1875440516835328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330172:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575539.330174:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575539.330176:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575539.330177:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575539.330179:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575539.330181:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516835328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575539.330184:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575539.330186:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575539.330187:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575539.330189:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575539.330192:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575539.330194:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575539.330197:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575539.330199:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575539.330203:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575539.330206:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516835328, portal 10 00000100:00000001:0.0:1788575539.330207:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575539.330208:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603414528 : -123400106137088 : ffff8fc4ac822a00) 00000100:00000040:0.0:1788575539.330214:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822a00 x1875440516835328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330220:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575539.330226:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575539.330228:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575539.330231:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516835328, offset 0 00000400:00000200:0.0:1788575539.330235:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575539.330239:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575539.330242:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575539.330245:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575539.330247:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575539.330249:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575539.330251:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575539.330254:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548178:548178:256:4294967295] 192.168.202.128@tcp LPNI seq info [548101:548101:8:4294967295] 00000400:00000200:0.0:1788575539.330257:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575539.330263:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575539.330267:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575539.330270:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575539.330274:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575539.330276:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1788575539.330286:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575539.330288:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575539.330291:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575539.330294:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575539.330297:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575539.330299:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575539.330301:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575539.330305:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822a00 x1875440516835328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330313:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575539.330315:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822a00 x1875440516835328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330321:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.330327:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575539.330331:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575539.330334:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575539.330337:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575539.330339:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310899226 00000100:00000001:0.0:1788575539.330341:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310903021 00000100:00000001:0.0:1788575539.330343:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310899226 00000100:00000001:0.0:1788575539.330345:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310903021 00000100:00000001:0.0:1788575539.330347:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575539.330349:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.330351:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575539.330354:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575539/real 1788575539] req@ffff8fc4ac822a00 x1875440516835328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575539.330361:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575539.330363:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575539.330364:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575539.330367:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575539.330368:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575539.330370:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575539.330373:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822a00 x1875440516835328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575539.330380:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822a00 x1875440516835328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330386:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575539.330388:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330389:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575539.330391:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575539.330394:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822a00 x1875440516835328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330400:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575539.330410:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822a00 x1875440516835328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330417:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575539.330419:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330421:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575539.330422:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330424:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575539.330426:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575539.330428:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575539.330431:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575539.330433:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575539.330436:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822a00 x1875440516835328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575539.330443:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516835328:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575539.330447:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575539.330450:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821500 x1875440516835456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330456:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575539.330457:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575539.330459:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575539.330460:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575539.330462:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575539.330465:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516835456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575539.330467:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575539.330469:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575539.330470:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575539.330472:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575539.330475:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575539.330477:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575539.330484:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575539.330486:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575539.330489:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575539.330492:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516835456, portal 4 00000100:00000001:0.0:1788575539.330494:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575539.330495:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603409152 : -123400106142464 : ffff8fc4ac821500) 00000100:00000040:0.0:1788575539.330500:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821500 x1875440516835456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330506:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575539.330508:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575539.330510:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575539.330512:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516835456, offset 0 00000400:00000200:0.0:1788575539.330515:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575539.330519:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575539.330522:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575539.330525:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575539.330527:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575539.330530:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575539.330532:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575539.330535:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548179:548179:256:4294967295] 192.168.202.128@tcp LPNI seq info [548102:548102:8:4294967295] 00000400:00000200:0.0:1788575539.330539:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575539.330545:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575539.330549:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575539.330552:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575539.330556:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575539.330558:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1788575539.330567:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575539.330569:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575539.330572:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575539.330575:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575539.330578:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575539.330583:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575539.330586:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575539.330589:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821500 x1875440516835456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330596:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575539.330599:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821500 x1875440516835456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330604:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.330606:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575539.330609:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575539.330612:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575539.330614:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575539.330616:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310899366 00000100:00000001:0.0:1788575539.330619:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310903021 00000100:00000001:0.0:1788575539.330620:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310899366 00000100:00000001:0.0:1788575539.330622:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310903021 00000100:00000001:0.0:1788575539.330624:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575539.330626:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.330627:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575539.330631:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575539/real 1788575539] req@ffff8fc4ac821500 x1875440516835456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575539.330638:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575539.330639:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575539.330640:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575539.330643:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575539.330644:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575539.330646:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575539.330649:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821500 x1875440516835456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575539.330656:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821500 x1875440516835456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330666:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575539.330668:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330670:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575539.330671:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575539.330675:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821500 x1875440516835456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330682:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575539.330686:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821500 x1875440516835456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330692:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575539.330695:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330696:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575539.330698:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330699:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575539.330701:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575539.330703:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575539.330707:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575539.330709:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575539.330713:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821500 x1875440516835456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575539.330720:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516835456:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575539.330724:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330726:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575539.330729:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823100 x1875440516835200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330735:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575539.330737:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575539.330744:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575539.330747:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575539.330749:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575539.330750:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575539.330752:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575539.330754:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862ac00. 02000000:00000001:0.0:1788575539.330756:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575539.330758:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575539.330759:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823100. 00000100:00000001:0.0:1788575539.330762:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575539.330764:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330765:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575539.330769:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822a00 x1875440516835328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330775:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575539.330776:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575539.330778:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575539.330780:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575539.330781:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575539.330783:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575539.330784:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575539.330785:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788575539.330787:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575539.330789:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575539.330790:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822a00. 00000100:00000001:0.0:1788575539.330792:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575539.330793:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330795:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575539.330798:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821500 x1875440516835456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330804:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575539.330805:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575539.330806:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575539.330825:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575539.330827:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575539.330828:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575539.330835:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575539.330837:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788575539.330840:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575539.330841:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575539.330842:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821500. 00000100:00000001:0.0:1788575539.330844:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575539.330846:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330847:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.330851:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575539.330852:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.330854:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575539.330856:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575539.330857:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575539.330861:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823b80 x1875440516835584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330867:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575539.330869:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575539.330871:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575539.330873:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575539.330875:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575539.330878:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516835584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575539.330880:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575539.330882:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575539.330884:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575539.330885:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575539.330888:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575539.330890:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575539.330893:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575539.330896:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575539.330899:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575539.330901:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516835584, portal 4 00000100:00000001:0.0:1788575539.330903:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575539.330904:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603419008 : -123400106132608 : ffff8fc4ac823b80) 00000100:00000040:0.0:1788575539.330909:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823b80 x1875440516835584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330925:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575539.330928:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575539.330930:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575539.330932:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516835584, offset 0 00000400:00000200:0.0:1788575539.330934:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575539.330937:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575539.330939:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575539.330941:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575539.330942:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575539.330944:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575539.330945:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575539.330947:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548180:548180:256:4294967295] 192.168.202.128@tcp LPNI seq info [548103:548103:8:4294967295] 00000400:00000200:0.0:1788575539.330949:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575539.330953:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575539.330956:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575539.330958:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575539.330960:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575539.330962:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1788575539.330968:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575539.330969:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575539.330971:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575539.330973:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575539.330975:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575539.330976:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575539.330977:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575539.330980:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823b80 x1875440516835584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330985:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575539.330986:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823b80 x1875440516835584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575564 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.330997:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.330997:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575539.331000:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575539.331001:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575539.331004:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575539.331005:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310899369 00000100:00000001:0.0:1788575539.331006:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310903021 00000100:00000001:0.0:1788575539.331008:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310899369 00000100:00000001:0.0:1788575539.331009:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310903021 00000100:00000001:0.0:1788575539.331010:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575539.331011:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.331012:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575539.331014:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575539/real 1788575539] req@ffff8fc4ac823b80 x1875440516835584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575539.331019:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575539.331020:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575539.331021:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575539.331023:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575539.331025:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575539.331027:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575539.331029:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823b80 x1875440516835584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575539.331037:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823b80 x1875440516835584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.331043:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575539.331044:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.331046:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575539.331047:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575539.331050:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823b80 x1875440516835584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.331062:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575539.331066:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823b80 x1875440516835584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.331072:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575539.331074:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.331075:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575539.331075:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.331077:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575539.331078:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575539.331080:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575539.331082:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575539.331083:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575539.331085:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823b80 x1875440516835584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575539.331090:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516835584:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575539.331091:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.331093:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575539.331094:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823b80 x1875440516835584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575564 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575539.331097:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575539.331098:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575539.331099:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575539.331101:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575539.331102:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575539.331103:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575539.331104:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575539.331105:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788575539.331107:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575539.331111:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575539.331112:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823b80. 00000100:00000001:0.0:1788575539.331113:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575539.331114:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.331115:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.331117:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575539.331118:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.331119:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575539.331132:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575539.840901:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575539.840908:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575539.840912:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575539.840918:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575539.840924:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575539.840929:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.840940:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575539.840944:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575539.840949:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575540.545014:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575540.545017:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575540.545020:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575540.864964:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575540.864979:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575540.864987:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575540.864997:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575540.865006:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575540.865023:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575540.865061:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575540.865066:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575540.865072:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575541.888931:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575541.888947:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575541.888955:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575541.888973:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575541.888993:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575541.889005:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575541.889035:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575541.889040:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575541.889062:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575542.592997:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575542.593004:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575542.593016:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575542.912962:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575542.912972:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575542.912976:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575542.912988:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575542.912998:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575542.913010:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575542.913025:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575542.913035:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575542.913042:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575543.168933:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310903021, at 4310903040 00000100:00000010:0.0:1788575543.168954:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575543.617037:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575543.617041:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575543.617044:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575543.936929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575543.936934:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575543.936939:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575543.936944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575543.936949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575543.936955:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575543.936974:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575543.936978:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575543.936981:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.449214:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575544.449224:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575544.449229:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575544.449246:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516835712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.449278:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575544.449282:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575544.449293:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575544.449296:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575544.449319:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575544.449331:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516835712:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575544.449343:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575544.449350:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575544.449355:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575544.449360:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575544.449367:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575544.449372:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575544.449388:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575544.449398:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575544.449408:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575544.449419:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516835712, portal 25 00000100:00000001:0.0:1788575544.449424:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575544.449428:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788575544.449440:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516835712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.449461:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575544.449474:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575544.449481:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575544.449490:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516835712, offset 0 00000400:00000200:0.0:1788575544.449510:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575544.449527:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575544.449547:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575544.449559:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575544.449570:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575544.449581:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575544.449591:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575544.449605:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548181:548181:256:4294967295] 192.168.202.128@tcp LPNI seq info [548104:548104:8:4294967295] 00000400:00000200:0.0:1788575544.449626:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575544.449665:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575544.449690:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575544.449712:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575544.449731:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575544.449745:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575544.449805:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575544.449860:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575544.449872:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575544.449882:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575544.449892:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575544.449900:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575544.449910:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575544.449921:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516835712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.449964:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575544.449980:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516835712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450027:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.450035:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575544.450047:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575544.450068:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575544.450077:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310908142 00000100:00000010:0.0:1788575544.450090:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575544.450102:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575544.450110:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310908142 00000100:00000010:0.0:1788575544.450118:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50f60. 00000100:00000001:0.0:1788575544.450129:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575544.450140:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575544.450146:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.450152:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575544.450162:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575544/real 1788575544] req@ffff8fc4ac822d80 x1875440516835712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575544.450227:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575544.450231:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575544.450237:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575544.450244:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575544.450248:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575544.450253:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575544.450265:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516835712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575544.450282:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516835712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450289:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575544.450291:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.450294:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575544.450296:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575544.450300:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516835712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450306:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575544.450310:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516835712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450317:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575544.450320:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.450321:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575544.450322:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.450325:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575544.450327:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575544.450331:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575544.450335:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575544.450337:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575544.450341:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516835712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575544.450355:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516835712:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575544.450360:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.450362:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575544.450365:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516835712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450370:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575544.450373:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575544.450376:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575544.450380:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575544.450383:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575544.450384:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575544.450387:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575544.450389:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628000. 02000000:00000001:0.0:1788575544.450392:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575544.450394:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575544.450396:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788575544.450400:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575544.450401:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.450404:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.450411:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575544.450413:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.450415:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575544.450417:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575544.450418:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575544.450422:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516835840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450428:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575544.450430:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575544.450432:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575544.450434:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575544.450436:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575544.450439:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516835840:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575544.450447:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575544.450449:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575544.450451:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575544.450453:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575544.450456:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575544.450457:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575544.450461:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575544.450464:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575544.450468:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575544.450471:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516835840, portal 10 00000100:00000001:0.0:1788575544.450473:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575544.450475:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788575544.450482:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516835840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450488:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575544.450490:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575544.450492:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575544.450494:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516835840, offset 0 00000400:00000200:0.0:1788575544.450498:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575544.450502:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575544.450505:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575544.450509:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575544.450512:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575544.450514:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575544.450516:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575544.450519:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548182:548182:256:4294967295] 192.168.202.128@tcp LPNI seq info [548105:548105:8:4294967295] 00000400:00000200:0.0:1788575544.450523:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575544.450529:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575544.450533:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575544.450536:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575544.450540:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575544.450548:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575544.450564:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575544.450566:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575544.450569:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575544.450572:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575544.450576:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575544.450577:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575544.450580:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575544.450583:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516835840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450590:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575544.450593:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516835840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450598:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.450599:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575544.450603:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575544.450606:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575544.450609:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575544.450611:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310904339 00000100:00000001:0.0:1788575544.450614:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310908142 00000100:00000001:0.0:1788575544.450616:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310904339 00000100:00000001:0.0:1788575544.450618:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310908142 00000100:00000001:0.0:1788575544.450620:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575544.450622:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.450623:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575544.450626:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575544/real 1788575544] req@ffff8fc4ac821f80 x1875440516835840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575544.450634:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575544.450635:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575544.450644:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575544.450646:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575544.450648:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575544.450650:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575544.450654:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516835840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575544.450662:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516835840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450668:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575544.450669:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.450671:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575544.450672:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575544.450676:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516835840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450682:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575544.450686:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516835840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450692:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575544.450695:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.450696:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575544.450697:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.450698:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575544.450700:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575544.450702:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575544.450704:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575544.450706:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575544.450708:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516835840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575544.450713:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516835840:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575544.450719:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575544.450722:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516835968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450727:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575544.450728:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575544.450729:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575544.450730:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575544.450732:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575544.450733:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516835968:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575544.450736:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575544.450737:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575544.450738:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575544.450739:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575544.450741:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575544.450743:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575544.450745:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575544.450746:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575544.450749:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575544.450751:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516835968, portal 10 00000100:00000001:0.0:1788575544.450752:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575544.450753:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788575544.450758:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516835968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450765:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575544.450767:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575544.450769:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575544.450771:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516835968, offset 0 00000400:00000200:0.0:1788575544.450775:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575544.450779:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575544.450782:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575544.450785:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575544.450792:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575544.450795:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575544.450797:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575544.450799:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548183:548183:256:4294967295] 192.168.202.128@tcp LPNI seq info [548106:548106:8:4294967295] 00000400:00000200:0.0:1788575544.450803:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575544.450835:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575544.450841:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575544.450845:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575544.450849:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575544.450851:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575544.450861:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575544.450862:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575544.450865:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575544.450868:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575544.450871:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575544.450872:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575544.450875:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575544.450879:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516835968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450886:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575544.450889:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516835968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450895:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.450896:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575544.450899:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575544.450902:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575544.450905:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575544.450907:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310904347 00000100:00000001:0.0:1788575544.450909:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310908142 00000100:00000001:0.0:1788575544.450911:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310904347 00000100:00000001:0.0:1788575544.450918:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310908142 00000100:00000001:0.0:1788575544.450921:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575544.450922:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.450924:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575544.450929:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575544/real 1788575544] req@ffff8fc4ac823480 x1875440516835968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575544.450937:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575544.450938:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575544.450940:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575544.450942:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575544.450943:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575544.450945:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575544.450948:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516835968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575544.450955:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516835968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450961:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575544.450962:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.450964:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575544.450965:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575544.450968:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516835968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450975:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575544.450978:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516835968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.450984:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575544.450987:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.450989:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575544.450990:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.450997:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575544.451000:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575544.451003:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575544.451007:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575544.451009:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575544.451013:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516835968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575544.451021:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516835968:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575544.451025:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575544.451029:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516836096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451036:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575544.451037:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575544.451039:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575544.451041:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575544.451043:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575544.451046:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516836096:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575544.451049:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575544.451051:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575544.451053:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575544.451054:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575544.451058:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575544.451060:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575544.451063:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575544.451066:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575544.451070:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575544.451073:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516836096, portal 4 00000100:00000001:0.0:1788575544.451075:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575544.451077:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788575544.451082:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516836096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451093:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575544.451095:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575544.451097:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575544.451099:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516836096, offset 0 00000400:00000200:0.0:1788575544.451103:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575544.451107:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575544.451111:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575544.451113:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575544.451115:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575544.451117:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575544.451119:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575544.451122:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548184:548184:256:4294967295] 192.168.202.128@tcp LPNI seq info [548107:548107:8:4294967295] 00000400:00000200:0.0:1788575544.451126:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575544.451132:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575544.451137:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575544.451140:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575544.451143:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575544.451146:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575544.451155:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575544.451157:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575544.451159:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575544.451162:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575544.451165:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575544.451166:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575544.451169:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575544.451172:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516836096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451179:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575544.451182:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516836096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451198:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.451200:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575544.451219:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575544.451222:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575544.451225:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575544.451227:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310904487 00000100:00000001:0.0:1788575544.451229:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310908142 00000100:00000001:0.0:1788575544.451231:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310904487 00000100:00000001:0.0:1788575544.451233:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310908142 00000100:00000001:0.0:1788575544.451235:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575544.451237:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.451238:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575544.451243:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575544/real 1788575544] req@ffff8fc4ac820700 x1875440516836096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575544.451250:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575544.451251:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575544.451253:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575544.451255:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575544.451257:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575544.451259:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575544.451262:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516836096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575544.451270:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516836096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451276:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575544.451277:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.451279:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575544.451280:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575544.451284:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516836096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451297:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575544.451302:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516836096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451308:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575544.451310:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.451312:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575544.451313:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.451315:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575544.451317:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575544.451319:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575544.451323:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575544.451325:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575544.451329:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516836096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575544.451337:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516836096:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575544.451341:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575544.451344:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516836224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451350:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575544.451352:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575544.451354:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575544.451356:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575544.451358:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575544.451361:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516836224:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575544.451364:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575544.451366:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575544.451372:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575544.451374:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575544.451377:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575544.451379:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575544.451383:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575544.451385:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575544.451388:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575544.451391:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516836224, portal 4 00000100:00000001:0.0:1788575544.451393:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575544.451394:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788575544.451400:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516836224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451406:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575544.451408:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575544.451410:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575544.451412:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516836224, offset 0 00000400:00000200:0.0:1788575544.451416:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575544.451420:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575544.451424:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575544.451427:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575544.451429:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575544.451431:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575544.451434:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575544.451436:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548185:548185:256:4294967295] 192.168.202.128@tcp LPNI seq info [548108:548108:8:4294967295] 00000400:00000200:0.0:1788575544.451441:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575544.451446:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575544.451451:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575544.451454:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575544.451457:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575544.451460:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575544.451470:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575544.451477:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575544.451480:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575544.451484:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575544.451487:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575544.451489:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575544.451492:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575544.451495:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516836224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451503:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575544.451506:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516836224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575569 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451511:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.451513:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575544.451516:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575544.451519:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575544.451523:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575544.451525:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310904489 00000100:00000001:0.0:1788575544.451527:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310908142 00000100:00000001:0.0:1788575544.451529:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310904489 00000100:00000001:0.0:1788575544.451531:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310908142 00000100:00000001:0.0:1788575544.451533:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575544.451535:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.451537:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575544.451541:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575544/real 1788575544] req@ffff8fc4ac822300 x1875440516836224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575544.451548:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575544.451549:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575544.451551:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575544.451553:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575544.451554:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575544.451562:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575544.451566:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516836224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575544.451573:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516836224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451580:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575544.451581:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.451583:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575544.451585:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575544.451588:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516836224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451595:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575544.451598:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516836224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451605:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575544.451607:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.451609:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575544.451610:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.451612:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575544.451613:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575544.451616:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575544.451619:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575544.451622:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575544.451626:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516836224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575544.451634:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516836224:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575544.451637:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.451639:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575544.451648:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516835840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451654:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575544.451656:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575544.451658:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575544.451661:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575544.451662:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575544.451664:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575544.451665:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575544.451667:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628800. 02000000:00000001:0.0:1788575544.451669:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575544.451671:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575544.451672:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788575544.451675:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575544.451676:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.451678:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575544.451681:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516835968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451688:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575544.451689:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575544.451691:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575544.451693:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575544.451694:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575544.451695:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575544.451697:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575544.451698:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a800. 02000000:00000001:0.0:1788575544.451701:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575544.451702:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575544.451703:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788575544.451706:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575544.451707:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.451708:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575544.451712:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516836096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451718:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575544.451725:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575544.451727:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575544.451729:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575544.451731:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575544.451732:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575544.451734:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575544.451735:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788575544.451737:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575544.451738:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575544.451739:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788575544.451742:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575544.451743:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.451744:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575544.451748:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516836224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575569 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575544.451754:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575544.451755:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575544.451757:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575544.451759:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575544.451760:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575544.451761:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575544.451763:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575544.451764:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788575544.451766:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575544.451767:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575544.451769:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788575544.451771:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575544.451772:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.451773:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.451778:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575544.451780:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.451782:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575544.451783:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575544.960921:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575544.960929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575544.960932:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575544.960939:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575544.960959:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575544.960966:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.960971:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575544.960978:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575544.960986:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575545.664924:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575545.664928:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575545.664931:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575545.984897:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575545.984904:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575545.984908:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575545.984914:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575545.984921:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575545.984927:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575545.984936:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575545.984943:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575545.984950:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575545.984987:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575545.984991:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575545.984995:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575545.984999:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575545.985011:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575545.985020:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575545.985024:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575545.985027:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575545.985030:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575547.008962:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575547.008969:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575547.008974:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575547.008986:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575547.008999:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575547.009008:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575547.009033:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575547.009037:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575547.009041:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575547.009099:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575547.009103:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575547.009105:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575547.009127:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575547.009130:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575547.009132:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575547.009134:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575547.009136:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575547.009139:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575547.713009:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575547.713012:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575547.713014:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575548.032921:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575548.032926:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575548.032930:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575548.032936:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575548.032942:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575548.032947:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575548.032967:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575548.032970:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575548.032973:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575548.032990:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575548.032993:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575548.032995:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575548.032998:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575548.033000:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575548.033003:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575548.033005:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575548.033007:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575548.033010:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575548.288930:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310908142, at 4310908160 00000100:00000010:0.0:1788575548.288952:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575548.737053:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575548.737055:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575548.737058:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575549.056981:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575549.056987:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575549.056991:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575549.056997:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575549.057003:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575549.057018:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.057020:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575549.057021:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.057024:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.569092:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575549.569098:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575549.569103:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575549.569113:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516836352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.569123:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575549.569126:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575549.569130:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575549.569133:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575549.569137:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575549.569143:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516836352:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575549.569149:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575549.569153:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575549.569157:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575549.569158:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575549.569164:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575549.569168:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575549.569182:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575549.569188:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575549.569195:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575549.569203:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516836352, portal 25 00000100:00000001:0.0:1788575549.569208:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575549.569211:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575549.569218:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516836352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.569227:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575549.569232:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575549.569235:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575549.569238:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516836352, offset 0 00000400:00000200:0.0:1788575549.569245:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575549.569265:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575549.569273:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575549.569280:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575549.569299:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575549.569304:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575549.569309:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575549.569313:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548186:548186:256:4294967295] 192.168.202.128@tcp LPNI seq info [548109:548109:8:4294967295] 00000400:00000200:0.0:1788575549.569320:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575549.569329:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575549.569335:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575549.569343:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575549.569350:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575549.569356:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575549.569381:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575549.569386:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575549.569395:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575549.569405:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575549.569412:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575549.569417:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575549.569424:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575549.569431:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516836352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.569449:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575549.569455:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516836352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.569465:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.569469:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575549.569474:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575549.569482:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575549.569486:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310913261 00000100:00000010:0.0:1788575549.569493:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575549.569508:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575549.569516:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310913261 00000100:00000010:0.0:1788575549.569522:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b501e0. 00000100:00000001:0.0:1788575549.569529:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575549.569536:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575549.569542:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.569548:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575549.569556:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575549/real 1788575549] req@ffff8fc4901a7800 x1875440516836352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575549.569583:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575549.569587:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575549.569591:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575549.569597:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575549.569601:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575549.569608:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575549.569616:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516836352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575549.569640:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516836352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.569658:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575549.569661:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.569666:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575549.569671:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575549.569680:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516836352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.569699:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575549.569710:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516836352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.569728:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575549.569734:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.569745:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575549.569748:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.569754:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575549.569758:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575549.569765:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575549.569774:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575549.569777:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575549.569783:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516836352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575549.569792:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516836352:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575549.569797:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.569800:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575549.569804:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516836352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.569851:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575549.569854:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575549.569857:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575549.569862:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575549.569863:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575549.569865:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575549.569866:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575549.569868:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575549.569871:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575549.569873:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575549.569874:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575549.569877:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575549.569878:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.569879:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.569884:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575549.569885:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.569886:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575549.569887:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575549.569889:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575549.569895:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516836480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.569900:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575549.569901:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575549.569902:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575549.569903:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575549.569904:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575549.569906:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516836480:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575549.569908:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575549.569909:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575549.569910:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575549.569912:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575549.569913:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575549.569915:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575549.569917:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575549.569919:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575549.569921:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575549.569923:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516836480, portal 10 00000100:00000001:0.0:1788575549.569925:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575549.569926:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575549.569930:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516836480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.569934:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575549.569935:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575549.569936:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575549.569938:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516836480, offset 0 00000400:00000200:0.0:1788575549.569940:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575549.569943:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575549.569945:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575549.569947:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575549.569948:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575549.569950:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575549.569954:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575549.569956:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548187:548187:256:4294967295] 192.168.202.128@tcp LPNI seq info [548110:548110:8:4294967295] 00000400:00000200:0.0:1788575549.569959:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575549.569963:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575549.569966:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575549.569968:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575549.569970:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575549.569972:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575549.569979:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575549.569980:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575549.569982:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575549.569984:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575549.569986:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575549.569987:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575549.569989:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575549.569991:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516836480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.569996:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575549.569997:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516836480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570001:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.570002:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575549.570004:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575549.570006:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575549.570008:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575549.570009:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310909459 00000100:00000001:0.0:1788575549.570011:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310913261 00000100:00000001:0.0:1788575549.570012:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310909459 00000100:00000001:0.0:1788575549.570014:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310913261 00000100:00000001:0.0:1788575549.570019:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575549.570020:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.570022:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575549.570024:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575549/real 1788575549] req@ffff8fc4901a5c00 x1875440516836480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575549.570029:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575549.570030:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575549.570031:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575549.570033:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575549.570033:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575549.570035:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575549.570037:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516836480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575549.570042:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516836480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570045:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575549.570046:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570047:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575549.570048:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575549.570050:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516836480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570053:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575549.570055:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516836480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570059:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575549.570060:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570061:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575549.570061:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570062:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575549.570064:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575549.570068:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575549.570070:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575549.570071:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575549.570073:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516836480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575549.570078:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516836480:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575549.570080:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575549.570082:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516836608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570085:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575549.570086:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575549.570087:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575549.570088:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575549.570089:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575549.570091:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516836608:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575549.570092:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575549.570093:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575549.570094:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575549.570095:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575549.570097:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575549.570098:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575549.570100:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575549.570101:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575549.570103:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575549.570105:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516836608, portal 10 00000100:00000001:0.0:1788575549.570106:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575549.570107:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575549.570110:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516836608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570120:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575549.570122:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575549.570123:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575549.570124:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516836608, offset 0 00000400:00000200:0.0:1788575549.570126:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575549.570129:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575549.570130:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575549.570132:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575549.570134:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575549.570135:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575549.570136:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575549.570138:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548188:548188:256:4294967295] 192.168.202.128@tcp LPNI seq info [548111:548111:8:4294967295] 00000400:00000200:0.0:1788575549.570140:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575549.570144:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575549.570146:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575549.570148:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575549.570150:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575549.570151:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575549.570157:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575549.570159:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575549.570160:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575549.570162:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575549.570164:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575549.570165:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575549.570167:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575549.570169:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516836608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570174:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575549.570177:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516836608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570193:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.570194:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575549.570198:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575549.570202:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575549.570206:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575549.570207:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310909466 00000100:00000001:0.0:1788575549.570210:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310913261 00000100:00000001:0.0:1788575549.570212:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310909466 00000100:00000001:0.0:1788575549.570214:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310913261 00000100:00000001:0.0:1788575549.570216:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575549.570218:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.570220:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575549.570224:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575549/real 1788575549] req@ffff8fc4901a5180 x1875440516836608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575549.570232:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575549.570233:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575549.570235:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575549.570237:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575549.570239:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575549.570241:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575549.570244:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516836608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575549.570251:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516836608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570256:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575549.570257:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570259:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575549.570260:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575549.570264:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516836608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570277:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575549.570294:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516836608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570300:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575549.570302:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570303:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575549.570304:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570306:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575549.570308:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575549.570311:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575549.570314:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575549.570317:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575549.570321:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516836608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575549.570329:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516836608:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575549.570333:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575549.570336:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516836736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570343:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575549.570344:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575549.570346:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575549.570348:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575549.570350:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575549.570353:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516836736:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575549.570356:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575549.570358:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575549.570364:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575549.570366:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575549.570369:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575549.570371:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575549.570375:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575549.570377:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575549.570381:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575549.570384:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516836736, portal 4 00000100:00000001:0.0:1788575549.570386:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575549.570387:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575549.570393:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516836736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570400:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575549.570402:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575549.570404:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575549.570406:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516836736, offset 0 00000400:00000200:0.0:1788575549.570410:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575549.570414:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575549.570417:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575549.570420:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575549.570423:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575549.570425:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575549.570427:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575549.570430:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548189:548189:256:4294967295] 192.168.202.128@tcp LPNI seq info [548112:548112:8:4294967295] 00000400:00000200:0.0:1788575549.570433:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575549.570439:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575549.570444:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575549.570447:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575549.570451:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575549.570454:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575549.570463:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575549.570471:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575549.570473:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575549.570477:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575549.570481:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575549.570482:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575549.570484:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575549.570486:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516836736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570491:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575549.570493:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516836736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570497:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.570498:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575549.570500:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575549.570502:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575549.570504:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575549.570506:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310909606 00000100:00000001:0.0:1788575549.570507:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310913261 00000100:00000001:0.0:1788575549.570509:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310909606 00000100:00000001:0.0:1788575549.570510:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310913261 00000100:00000001:0.0:1788575549.570511:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575549.570512:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.570514:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575549.570516:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575549/real 1788575549] req@ffff8fc4901a7100 x1875440516836736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575549.570520:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575549.570521:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575549.570522:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575549.570523:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575549.570524:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575549.570529:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575549.570531:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516836736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575549.570536:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516836736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570539:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575549.570540:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570541:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575549.570542:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575549.570544:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516836736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570548:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575549.570550:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516836736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570553:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575549.570555:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570556:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575549.570556:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570557:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575549.570558:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575549.570560:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575549.570561:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575549.570563:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575549.570565:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516836736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575549.570569:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516836736:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575549.570571:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575549.570573:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516836864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570579:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575549.570580:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575549.570582:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575549.570583:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575549.570584:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575549.570586:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516836864:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575549.570587:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575549.570588:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575549.570590:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575549.570591:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575549.570592:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575549.570593:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575549.570595:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575549.570597:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575549.570599:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575549.570601:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516836864, portal 4 00000100:00000001:0.0:1788575549.570602:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575549.570603:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575549.570606:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516836864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570610:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575549.570611:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575549.570613:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575549.570614:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516836864, offset 0 00000400:00000200:0.0:1788575549.570616:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575549.570618:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575549.570620:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575549.570622:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575549.570623:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575549.570624:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575549.570626:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575549.570630:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548190:548190:256:4294967295] 192.168.202.128@tcp LPNI seq info [548113:548113:8:4294967295] 00000400:00000200:0.0:1788575549.570633:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575549.570638:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575549.570643:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575549.570646:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575549.570649:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575549.570651:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575549.570660:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575549.570662:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575549.570665:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575549.570668:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575549.570670:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575549.570672:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575549.570675:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575549.570678:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516836864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570686:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575549.570688:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516836864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575574 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570692:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.570693:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575549.570695:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575549.570697:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575549.570699:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575549.570700:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310909608 00000100:00000001:0.0:1788575549.570702:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310913261 00000100:00000001:0.0:1788575549.570703:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310909608 00000100:00000001:0.0:1788575549.570705:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310913261 00000100:00000001:0.0:1788575549.570706:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575549.570711:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.570712:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575549.570714:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575549/real 1788575549] req@ffff8fc4901a4a80 x1875440516836864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575549.570720:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575549.570721:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575549.570722:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575549.570723:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575549.570724:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575549.570726:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575549.570727:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516836864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575549.570732:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516836864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570736:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575549.570737:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570737:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575549.570738:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575549.570740:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516836864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570743:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575549.570746:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516836864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570749:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575549.570750:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570751:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575549.570752:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570753:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575549.570754:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575549.570755:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575549.570759:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575549.570761:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575549.570763:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516836864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575549.570767:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516836864:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575549.570769:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570770:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575549.570772:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516836480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570776:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575549.570777:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575549.570778:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575549.570780:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575549.570781:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575549.570782:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575549.570783:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575549.570784:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575549.570786:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575549.570787:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575549.570788:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575549.570791:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575549.570791:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570792:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575549.570794:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516836608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570798:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575549.570798:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575549.570799:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575549.570801:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575549.570801:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575549.570802:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575549.570803:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575549.570806:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575549.570824:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575549.570825:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575549.570827:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575549.570829:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575549.570831:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570833:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575549.570837:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516836736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570843:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575549.570844:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575549.570846:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575549.570848:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575549.570849:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575549.570850:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575549.570853:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575549.570854:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575549.570856:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575549.570857:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575549.570858:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575549.570860:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575549.570861:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570863:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575549.570866:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516836864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575574 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575549.570872:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575549.570873:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575549.570875:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575549.570877:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575549.570878:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575549.570879:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575549.570880:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575549.570881:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575549.570884:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575549.570885:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575549.570886:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575549.570895:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575549.570896:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570898:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575549.570901:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575549.570901:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575549.570903:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575549.570904:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575550.080899:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575550.080904:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575550.080907:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575550.080915:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575550.080921:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575550.080925:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575550.080944:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575550.080946:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575550.080949:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575550.080964:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575550.080967:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575550.080968:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575550.080970:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575550.080973:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575550.080976:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575550.080978:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575550.080980:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575550.080982:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575550.784994:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575550.784998:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575550.785001:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575551.104912:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575551.104919:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575551.104922:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575551.104927:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575551.104932:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575551.104936:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575551.104939:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575551.104943:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575551.104947:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575551.104962:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575551.104980:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575551.104981:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575551.104985:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575551.104989:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575551.104990:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575551.104992:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575551.104993:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575551.104995:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575552.128898:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575552.128904:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575552.128909:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575552.128916:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575552.128922:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575552.128926:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575552.128931:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575552.128937:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575552.128941:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575552.128975:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575552.128979:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575552.128981:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575552.128983:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575552.128985:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575552.128988:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575552.128990:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575552.128992:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575552.128994:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575552.833211:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575552.833215:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575552.833218:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575553.152923:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575553.152930:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575553.152934:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575553.152939:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575553.152956:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575553.152961:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575553.152964:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575553.152966:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575553.152969:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575553.408912:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310913261, at 4310913280 00000100:00000010:0.0:1788575553.408935:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575553.857044:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575553.857047:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575553.857049:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575554.177014:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575554.177017:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575554.177018:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575554.177021:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575554.177023:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575554.177025:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.177030:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575554.177032:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.177034:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.177067:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575554.177069:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575554.177070:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575554.177072:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575554.177073:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575554.177075:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.177077:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575554.177078:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.177080:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.689170:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575554.689175:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575554.689181:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575554.689190:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7100 x1875440516836992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.689206:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575554.689209:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575554.689216:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575554.689219:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575554.689222:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575554.689228:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516836992:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575554.689233:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575554.689243:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575554.689246:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575554.689250:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575554.689255:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575554.689257:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575554.689263:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575554.689267:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575554.689274:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575554.689281:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516836992, portal 25 00000100:00000001:0.0:1788575554.689284:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575554.689288:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605333248 : -123400104218368 : ffff8fc4ac9f7100) 00000100:00000040:0.0:1788575554.689299:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7100 x1875440516836992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.689317:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575554.689322:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575554.689324:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575554.689327:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516836992, offset 0 00000400:00000200:0.0:1788575554.689335:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575554.689344:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575554.689350:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575554.689355:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575554.689371:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575554.689375:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575554.689379:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575554.689382:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548191:548191:256:4294967295] 192.168.202.128@tcp LPNI seq info [548114:548114:8:4294967295] 00000400:00000200:0.0:1788575554.689387:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575554.689395:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575554.689399:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575554.689405:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575554.689410:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575554.689413:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575554.689424:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575554.689430:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575554.689437:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575554.689446:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575554.689453:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575554.689457:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575554.689464:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575554.689471:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7100 x1875440516836992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.689485:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575554.689491:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7100 x1875440516836992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.689506:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.689510:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575554.689517:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575554.689523:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575554.689526:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310918381 00000100:00000010:0.0:1788575554.689533:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575554.689542:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575554.689549:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310918381 00000100:00000010:0.0:1788575554.689556:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b501e0. 00000100:00000001:0.0:1788575554.689562:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575554.689568:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575554.689573:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.689578:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575554.689587:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575554/real 1788575554] req@ffff8fc4ac9f7100 x1875440516836992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575554.689603:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575554.689606:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575554.689610:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575554.689613:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575554.689615:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575554.689618:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575554.689627:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7100 x1875440516836992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575554.689645:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7100 x1875440516836992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.689660:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575554.689662:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.689666:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575554.689668:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575554.689674:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7100 x1875440516836992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.689684:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575554.689690:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7100 x1875440516836992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.689694:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575554.689696:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.689699:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575554.689701:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.689703:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575554.689707:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575554.689710:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575554.689715:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575554.689718:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575554.689723:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7100 x1875440516836992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575554.689735:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516836992:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575554.689741:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.689746:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575554.689752:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7100 x1875440516836992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.689773:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575554.689776:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575554.689779:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575554.689784:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575554.689787:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575554.689790:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575554.689793:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575554.689796:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36c00. 02000000:00000001:0.0:1788575554.689801:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575554.689803:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575554.689806:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7100. 00000100:00000001:0.0:1788575554.689827:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575554.689828:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.689830:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.689836:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575554.689840:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.689845:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575554.689847:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575554.689850:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575554.689858:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4000 x1875440516837120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.689872:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575554.689876:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575554.689880:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575554.689884:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575554.689889:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575554.689893:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516837120:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575554.689900:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575554.689902:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575554.689906:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575554.689910:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575554.689916:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575554.689920:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575554.689926:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575554.689936:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575554.689943:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575554.689948:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516837120, portal 10 00000100:00000001:0.0:1788575554.689953:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575554.689955:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605320704 : -123400104230912 : ffff8fc4ac9f4000) 00000100:00000040:0.0:1788575554.689966:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4000 x1875440516837120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.689979:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575554.689986:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575554.689989:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575554.689993:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516837120, offset 0 00000400:00000200:0.0:1788575554.690002:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575554.690009:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575554.690015:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575554.690027:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575554.690032:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575554.690034:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575554.690036:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575554.690039:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548192:548192:256:4294967295] 192.168.202.128@tcp LPNI seq info [548115:548115:8:4294967295] 00000400:00000200:0.0:1788575554.690043:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575554.690049:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575554.690054:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575554.690060:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575554.690067:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575554.690071:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575554.690095:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575554.690099:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575554.690102:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575554.690109:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575554.690116:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575554.690121:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575554.690135:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575554.690144:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4000 x1875440516837120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690162:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575554.690167:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4000 x1875440516837120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690184:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.690188:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575554.690194:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575554.690199:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575554.690207:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575554.690211:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310914579 00000100:00000001:0.0:1788575554.690216:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310918381 00000100:00000001:0.0:1788575554.690222:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310914579 00000100:00000001:0.0:1788575554.690226:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310918381 00000100:00000001:0.0:1788575554.690228:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575554.690232:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.690236:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575554.690243:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575554/real 1788575554] req@ffff8fc4ac9f4000 x1875440516837120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575554.690254:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575554.690256:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575554.690258:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575554.690260:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575554.690262:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575554.690264:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575554.690267:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4000 x1875440516837120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575554.690273:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4000 x1875440516837120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690289:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575554.690291:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690293:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575554.690294:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575554.690298:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4000 x1875440516837120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690303:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575554.690307:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4000 x1875440516837120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690313:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575554.690315:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690317:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575554.690318:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690320:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575554.690322:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575554.690324:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575554.690327:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575554.690329:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575554.690332:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4000 x1875440516837120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575554.690339:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516837120:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575554.690343:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575554.690346:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6680 x1875440516837248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690352:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575554.690353:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575554.690386:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575554.690388:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575554.690390:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575554.690393:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516837248:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575554.690397:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575554.690399:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575554.690400:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575554.690402:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575554.690405:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575554.690407:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575554.690410:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575554.690412:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575554.690416:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575554.690419:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516837248, portal 10 00000100:00000001:0.0:1788575554.690421:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575554.690422:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605330560 : -123400104221056 : ffff8fc4ac9f6680) 00000100:00000040:0.0:1788575554.690427:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6680 x1875440516837248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690433:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575554.690435:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575554.690437:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575554.690439:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516837248, offset 0 00000400:00000200:0.0:1788575554.690442:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575554.690446:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575554.690449:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575554.690452:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575554.690455:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575554.690457:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575554.690459:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575554.690462:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548193:548193:256:4294967295] 192.168.202.128@tcp LPNI seq info [548116:548116:8:4294967295] 00000400:00000200:0.0:1788575554.690465:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575554.690471:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575554.690481:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575554.690484:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575554.690487:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575554.690490:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575554.690500:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575554.690502:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575554.690504:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575554.690508:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575554.690511:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575554.690512:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575554.690515:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575554.690518:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6680 x1875440516837248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690524:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575554.690526:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6680 x1875440516837248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690530:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.690531:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575554.690533:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575554.690535:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575554.690537:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575554.690539:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310914586 00000100:00000001:0.0:1788575554.690540:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310918381 00000100:00000001:0.0:1788575554.690542:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310914586 00000100:00000001:0.0:1788575554.690543:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310918381 00000100:00000001:0.0:1788575554.690544:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575554.690545:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.690546:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575554.690548:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575554/real 1788575554] req@ffff8fc4ac9f6680 x1875440516837248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575554.690557:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575554.690558:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575554.690560:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575554.690561:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575554.690562:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575554.690563:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575554.690565:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6680 x1875440516837248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575554.690570:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6680 x1875440516837248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690574:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575554.690574:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690575:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575554.690576:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575554.690578:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6680 x1875440516837248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690581:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575554.690583:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6680 x1875440516837248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690587:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575554.690588:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690589:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575554.690590:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690591:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575554.690593:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575554.690594:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575554.690596:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575554.690597:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575554.690602:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6680 x1875440516837248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575554.690606:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516837248:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575554.690609:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690610:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575554.690611:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4000 x1875440516837120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690615:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575554.690615:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575554.690617:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575554.690618:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575554.690619:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575554.690620:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575554.690621:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575554.690622:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34400. 02000000:00000001:0.0:1788575554.690624:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575554.690625:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575554.690626:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4000. 00000100:00000001:0.0:1788575554.690627:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575554.690628:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690628:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575554.690630:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6680 x1875440516837248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690634:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575554.690634:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575554.690635:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575554.690636:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575554.690637:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575554.690638:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575554.690639:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575554.690639:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788575554.690640:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575554.690644:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575554.690645:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6680. 00000100:00000001:0.0:1788575554.690647:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575554.690648:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690649:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690651:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575554.690652:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.690653:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575554.690654:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575554.690655:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575554.690657:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7480 x1875440516837376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690661:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575554.690662:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575554.690663:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575554.690665:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575554.690666:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575554.690667:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516837376:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575554.690669:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575554.690670:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575554.690671:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575554.690672:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575554.690674:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575554.690675:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575554.690677:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575554.690678:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575554.690680:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575554.690682:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516837376, portal 4 00000100:00000001:0.0:1788575554.690683:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575554.690684:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605334144 : -123400104217472 : ffff8fc4ac9f7480) 00000100:00000040:0.0:1788575554.690688:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7480 x1875440516837376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690691:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575554.690693:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575554.690696:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575554.690698:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516837376, offset 0 00000400:00000200:0.0:1788575554.690700:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575554.690703:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575554.690705:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575554.690707:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575554.690708:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575554.690710:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575554.690711:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575554.690713:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548194:548194:256:4294967295] 192.168.202.128@tcp LPNI seq info [548117:548117:8:4294967295] 00000400:00000200:0.0:1788575554.690715:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575554.690719:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575554.690721:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575554.690723:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575554.690725:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575554.690727:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575554.690736:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575554.690737:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575554.690739:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575554.690741:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575554.690743:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575554.690744:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575554.690746:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575554.690748:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7480 x1875440516837376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690752:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575554.690754:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7480 x1875440516837376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690758:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.690759:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575554.690763:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575554.690765:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575554.690767:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575554.690768:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310914726 00000100:00000001:0.0:1788575554.690770:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310918381 00000100:00000001:0.0:1788575554.690771:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310914726 00000100:00000001:0.0:1788575554.690772:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310918381 00000100:00000001:0.0:1788575554.690773:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575554.690774:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.690776:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575554.690777:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575554/real 1788575554] req@ffff8fc4ac9f7480 x1875440516837376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575554.690782:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575554.690783:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575554.690784:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575554.690785:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575554.690786:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575554.690787:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575554.690789:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7480 x1875440516837376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575554.690793:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7480 x1875440516837376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690796:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575554.690797:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690799:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575554.690799:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575554.690801:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7480 x1875440516837376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690804:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575554.690806:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7480 x1875440516837376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690834:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575554.690837:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690838:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575554.690840:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.690841:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575554.690843:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575554.690846:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575554.690849:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575554.690852:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575554.690856:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7480 x1875440516837376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575554.690864:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516837376:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575554.690867:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575554.690870:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5c00 x1875440516837504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690877:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575554.690878:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575554.690881:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575554.690883:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575554.690884:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575554.690887:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516837504:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575554.690891:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575554.690892:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575554.690894:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575554.690896:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575554.690899:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575554.690901:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575554.690904:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575554.690911:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575554.690915:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575554.690918:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516837504, portal 4 00000100:00000001:0.0:1788575554.690920:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575554.690921:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605327872 : -123400104223744 : ffff8fc4ac9f5c00) 00000100:00000040:0.0:1788575554.690928:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5c00 x1875440516837504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.690934:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575554.690937:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575554.690939:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575554.690941:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516837504, offset 0 00000400:00000200:0.0:1788575554.690945:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575554.690949:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575554.690952:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575554.690954:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575554.690957:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575554.690959:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575554.690961:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575554.690963:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548195:548195:256:4294967295] 192.168.202.128@tcp LPNI seq info [548118:548118:8:4294967295] 00000400:00000200:0.0:1788575554.690967:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575554.690973:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575554.690977:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575554.690979:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575554.690982:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575554.690983:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575554.690990:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575554.690991:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575554.690993:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575554.690995:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575554.690997:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575554.690998:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575554.691004:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575554.691007:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5c00 x1875440516837504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.691011:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575554.691013:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5c00 x1875440516837504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575579 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.691017:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.691018:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575554.691020:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575554.691022:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575554.691024:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575554.691025:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310914729 00000100:00000001:0.0:1788575554.691028:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310918381 00000100:00000001:0.0:1788575554.691029:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310914729 00000100:00000001:0.0:1788575554.691031:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310918381 00000100:00000001:0.0:1788575554.691033:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575554.691035:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.691037:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575554.691040:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575554/real 1788575554] req@ffff8fc4ac9f5c00 x1875440516837504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575554.691048:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575554.691049:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575554.691051:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575554.691053:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575554.691055:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575554.691057:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575554.691060:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5c00 x1875440516837504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575554.691067:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5c00 x1875440516837504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.691084:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575554.691086:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.691088:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575554.691089:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575554.691093:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5c00 x1875440516837504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.691099:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575554.691103:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5c00 x1875440516837504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.691109:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575554.691111:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.691113:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575554.691114:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.691116:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575554.691117:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575554.691120:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575554.691123:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575554.691126:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575554.691130:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5c00 x1875440516837504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575554.691137:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516837504:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575554.691140:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.691142:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575554.691145:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7480 x1875440516837376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.691151:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575554.691152:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575554.691159:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575554.691162:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575554.691164:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575554.691165:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575554.691167:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575554.691169:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35400. 02000000:00000001:0.0:1788575554.691171:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575554.691172:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575554.691173:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7480. 00000100:00000001:0.0:1788575554.691176:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575554.691177:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.691179:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575554.691182:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5c00 x1875440516837504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575579 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575554.691188:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575554.691189:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575554.691191:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575554.691193:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575554.691194:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575554.691195:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575554.691197:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575554.691198:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788575554.691200:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575554.691202:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575554.691203:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5c00. 00000100:00000001:0.0:1788575554.691205:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575554.691206:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.691208:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575554.691212:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575554.691213:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575554.691215:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575554.691216:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575555.200969:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575555.200985:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575555.200994:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575555.201006:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575555.201019:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575555.201286:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575555.201288:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575555.201290:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575555.201292:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575555.905013:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575555.905016:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575555.905019:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575556.224977:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575556.224988:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575556.224992:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575556.225000:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575556.225011:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575556.225017:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575556.225025:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575556.225030:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575556.225046:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575557.248904:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575557.248907:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575557.248911:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575557.248916:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575557.248921:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575557.248928:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575557.248941:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575557.248944:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575557.248947:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575557.953018:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575557.953021:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575557.953023:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575558.272950:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575558.272958:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575558.272962:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575558.272968:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575558.272974:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575558.272979:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575558.272992:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575558.272996:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575558.273000:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575558.528927:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310918381, at 4310918400 00000100:00000010:0.0:1788575558.528961:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575558.977041:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575558.977045:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575558.977048:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575559.296896:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575559.296904:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575559.296909:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575559.296916:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575559.296920:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575559.296923:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.296930:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575559.296934:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.296939:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.296966:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575559.296970:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575559.296972:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575559.296974:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575559.296977:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575559.296979:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.296982:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575559.296983:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.296985:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.809123:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575559.809129:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575559.809131:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575559.809137:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5500 x1875440516837632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.809146:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575559.809149:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575559.809156:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575559.809160:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575559.809167:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575559.809174:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516837632:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575559.809183:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575559.809188:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575559.809199:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575559.809203:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575559.809208:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575559.809211:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575559.809227:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575559.809232:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575559.809240:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575559.809247:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516837632, portal 25 00000100:00000001:0.0:1788575559.809252:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575559.809255:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605326080 : -123400104225536 : ffff8fc4ac9f5500) 00000100:00000040:0.0:1788575559.809264:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5500 x1875440516837632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.809275:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575559.809281:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575559.809283:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575559.809285:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516837632, offset 0 00000400:00000200:0.0:1788575559.809295:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575559.809304:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575559.809312:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575559.809318:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575559.809324:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575559.809331:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575559.809335:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575559.809342:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548196:548196:256:4294967295] 192.168.202.128@tcp LPNI seq info [548119:548119:8:4294967295] 00000400:00000200:0.0:1788575559.809353:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575559.809367:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575559.809376:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575559.809384:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575559.809393:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575559.809397:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788575559.809415:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575559.809425:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575559.809433:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575559.809452:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575559.809459:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575559.809462:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575559.809468:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575559.809474:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5500 x1875440516837632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.809489:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575559.809494:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5500 x1875440516837632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.809504:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.809507:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575559.809513:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575559.809520:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575559.809525:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310923501 00000100:00000010:0.0:1788575559.809531:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575559.809539:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575559.809546:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310923501 00000100:00000010:0.0:1788575559.809549:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50f60. 00000100:00000001:0.0:1788575559.809554:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575559.809559:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575559.809563:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.809566:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575559.809572:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575559/real 1788575559] req@ffff8fc4ac9f5500 x1875440516837632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575559.809587:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575559.809590:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575559.809593:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575559.809596:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575559.809599:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575559.809603:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575559.809616:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5500 x1875440516837632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575559.809626:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5500 x1875440516837632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.809634:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575559.809637:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.809640:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575559.809643:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575559.809651:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5500 x1875440516837632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.809663:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575559.809672:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5500 x1875440516837632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.809681:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575559.809685:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.809689:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575559.809692:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.809696:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575559.809698:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575559.809702:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575559.809708:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575559.809711:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575559.809716:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5500 x1875440516837632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575559.809725:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516837632:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575559.809733:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.809737:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575559.809742:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5500 x1875440516837632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.809775:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575559.809778:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575559.809782:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575559.809788:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575559.809791:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575559.809794:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575559.809798:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575559.809801:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788575559.809806:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575559.809842:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575559.809848:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5500. 00000100:00000001:0.0:1788575559.809855:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575559.809859:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.809864:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.809873:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575559.809876:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.809879:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575559.809881:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575559.809883:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575559.809889:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4380 x1875440516837760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.809907:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575559.809909:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575559.809913:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575559.809916:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575559.809918:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575559.809922:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516837760:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575559.809925:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575559.809927:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575559.809929:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575559.809931:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575559.809933:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575559.809935:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575559.809938:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575559.809944:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575559.809947:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575559.809949:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516837760, portal 10 00000100:00000001:0.0:1788575559.809950:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575559.809951:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605321600 : -123400104230016 : ffff8fc4ac9f4380) 00000100:00000040:0.0:1788575559.809955:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4380 x1875440516837760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.809959:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575559.809961:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575559.809962:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575559.809964:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516837760, offset 0 00000400:00000200:0.0:1788575559.809966:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575559.809968:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575559.809970:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575559.809972:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575559.809974:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575559.809976:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575559.809978:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575559.809981:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548197:548197:256:4294967295] 192.168.202.128@tcp LPNI seq info [548120:548120:8:4294967295] 00000400:00000200:0.0:1788575559.809985:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575559.809991:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575559.809995:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575559.809998:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575559.810002:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575559.810005:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788575559.810015:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575559.810017:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575559.810026:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575559.810028:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575559.810031:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575559.810032:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575559.810038:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575559.810040:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4380 x1875440516837760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810045:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575559.810047:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4380 x1875440516837760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810051:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.810052:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575559.810054:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575559.810057:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575559.810059:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575559.810060:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310919699 00000100:00000001:0.0:1788575559.810063:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310923501 00000100:00000001:0.0:1788575559.810064:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310919699 00000100:00000001:0.0:1788575559.810065:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310923501 00000100:00000001:0.0:1788575559.810066:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575559.810067:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.810068:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575559.810071:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575559/real 1788575559] req@ffff8fc4ac9f4380 x1875440516837760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575559.810075:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575559.810076:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575559.810077:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575559.810079:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575559.810080:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575559.810081:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575559.810083:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4380 x1875440516837760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575559.810088:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4380 x1875440516837760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810097:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575559.810098:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810099:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575559.810100:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575559.810103:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4380 x1875440516837760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810106:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575559.810109:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4380 x1875440516837760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810113:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575559.810114:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810115:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575559.810116:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810117:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575559.810118:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575559.810119:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575559.810121:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575559.810122:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575559.810124:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4380 x1875440516837760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575559.810128:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516837760:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575559.810131:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575559.810132:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6300 x1875440516837888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810136:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575559.810137:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575559.810141:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575559.810142:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575559.810143:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575559.810145:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516837888:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575559.810147:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575559.810148:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575559.810149:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575559.810151:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575559.810153:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575559.810155:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575559.810158:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575559.810160:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575559.810164:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575559.810166:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516837888, portal 10 00000100:00000001:0.0:1788575559.810168:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575559.810169:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605329664 : -123400104221952 : ffff8fc4ac9f6300) 00000100:00000040:0.0:1788575559.810175:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6300 x1875440516837888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810181:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575559.810183:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575559.810185:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575559.810187:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516837888, offset 0 00000400:00000200:0.0:1788575559.810191:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575559.810195:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575559.810198:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575559.810200:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575559.810202:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575559.810204:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575559.810205:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575559.810207:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548198:548198:256:4294967295] 192.168.202.128@tcp LPNI seq info [548121:548121:8:4294967295] 00000400:00000200:0.0:1788575559.810209:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575559.810213:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575559.810220:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575559.810222:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575559.810225:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575559.810226:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788575559.810233:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575559.810234:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575559.810236:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575559.810238:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575559.810239:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575559.810240:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575559.810242:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575559.810245:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6300 x1875440516837888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810249:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575559.810251:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6300 x1875440516837888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810255:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.810256:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575559.810258:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575559.810260:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575559.810262:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575559.810263:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310919706 00000100:00000001:0.0:1788575559.810265:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310923501 00000100:00000001:0.0:1788575559.810266:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310919706 00000100:00000001:0.0:1788575559.810267:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310923501 00000100:00000001:0.0:1788575559.810268:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575559.810270:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.810271:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575559.810273:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575559/real 1788575559] req@ffff8fc4ac9f6300 x1875440516837888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575559.810281:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575559.810282:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575559.810283:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575559.810284:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575559.810285:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575559.810286:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575559.810288:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6300 x1875440516837888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575559.810293:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6300 x1875440516837888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810296:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575559.810297:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810298:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575559.810299:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575559.810301:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6300 x1875440516837888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810305:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575559.810307:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6300 x1875440516837888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810311:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575559.810312:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810313:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575559.810313:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810314:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575559.810316:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575559.810318:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575559.810319:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575559.810320:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575559.810327:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6300 x1875440516837888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575559.810332:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516837888:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575559.810334:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575559.810335:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7b80 x1875440516838016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810339:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575559.810340:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575559.810341:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575559.810342:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575559.810343:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575559.810345:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516838016:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575559.810346:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575559.810348:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575559.810349:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575559.810350:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575559.810351:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575559.810352:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575559.810355:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575559.810356:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575559.810358:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575559.810360:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516838016, portal 4 00000100:00000001:0.0:1788575559.810361:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575559.810362:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335936 : -123400104215680 : ffff8fc4ac9f7b80) 00000100:00000040:0.0:1788575559.810365:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7b80 x1875440516838016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810369:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575559.810370:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575559.810371:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575559.810373:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516838016, offset 0 00000400:00000200:0.0:1788575559.810394:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575559.810397:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575559.810399:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575559.810401:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575559.810402:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575559.810404:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575559.810405:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575559.810407:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548199:548199:256:4294967295] 192.168.202.128@tcp LPNI seq info [548122:548122:8:4294967295] 00000400:00000200:0.0:1788575559.810410:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575559.810413:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575559.810416:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575559.810418:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575559.810420:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575559.810422:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788575559.810428:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575559.810429:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575559.810431:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575559.810433:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575559.810435:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575559.810436:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575559.810437:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575559.810439:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7b80 x1875440516838016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810455:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575559.810457:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7b80 x1875440516838016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810461:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.810462:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575559.810464:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575559.810466:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575559.810468:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575559.810473:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310919846 00000100:00000001:0.0:1788575559.810475:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310923501 00000100:00000001:0.0:1788575559.810476:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310919846 00000100:00000001:0.0:1788575559.810477:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310923501 00000100:00000001:0.0:1788575559.810478:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575559.810480:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.810481:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575559.810483:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575559/real 1788575559] req@ffff8fc4ac9f7b80 x1875440516838016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575559.810488:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575559.810489:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575559.810490:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575559.810491:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575559.810492:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575559.810494:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575559.810495:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7b80 x1875440516838016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575559.810500:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7b80 x1875440516838016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810504:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575559.810505:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810506:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575559.810506:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575559.810510:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7b80 x1875440516838016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810516:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575559.810519:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7b80 x1875440516838016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810525:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575559.810532:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810533:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575559.810534:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810536:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575559.810538:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575559.810540:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575559.810544:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575559.810546:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575559.810550:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7b80 x1875440516838016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575559.810557:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516838016:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575559.810561:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810562:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575559.810566:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4380 x1875440516837760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810572:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575559.810573:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575559.810575:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575559.810578:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575559.810579:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575559.810580:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575559.810582:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575559.810584:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37c00. 02000000:00000001:0.0:1788575559.810587:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575559.810588:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575559.810589:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4380. 00000100:00000001:0.0:1788575559.810592:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575559.810593:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810595:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575559.810598:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6300 x1875440516837888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810615:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575559.810616:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575559.810618:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575559.810620:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575559.810621:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575559.810623:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575559.810624:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575559.810626:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788575559.810628:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575559.810629:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575559.810630:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6300. 00000100:00000001:0.0:1788575559.810632:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575559.810633:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810634:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575559.810637:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7b80 x1875440516838016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810643:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575559.810644:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575559.810645:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575559.810647:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575559.810648:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575559.810649:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575559.810651:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575559.810652:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35400. 02000000:00000001:0.0:1788575559.810654:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575559.810655:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575559.810656:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7b80. 00000100:00000001:0.0:1788575559.810658:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575559.810659:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810661:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810664:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575559.810665:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.810667:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575559.810668:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575559.810669:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575559.810676:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4a80 x1875440516838144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810682:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575559.810683:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575559.810685:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575559.810687:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575559.810689:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575559.810691:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516838144:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575559.810694:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575559.810696:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575559.810697:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575559.810699:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575559.810701:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575559.810703:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575559.810706:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575559.810709:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575559.810712:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575559.810714:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516838144, portal 4 00000100:00000001:0.0:1788575559.810716:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575559.810718:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605323392 : -123400104228224 : ffff8fc4ac9f4a80) 00000100:00000040:0.0:1788575559.810723:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4a80 x1875440516838144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810729:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575559.810732:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575559.810734:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575559.810736:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516838144, offset 0 00000400:00000200:0.0:1788575559.810740:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575559.810744:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575559.810747:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575559.810750:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575559.810752:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575559.810754:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575559.810756:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575559.810764:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548200:548200:256:4294967295] 192.168.202.128@tcp LPNI seq info [548123:548123:8:4294967295] 00000400:00000200:0.0:1788575559.810768:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575559.810774:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575559.810779:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575559.810782:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575559.810785:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575559.810788:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788575559.810798:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575559.810800:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56d00. 00000400:00000200:0.0:1788575559.810802:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575559.810805:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575559.810824:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9220 00000400:00000010:0.0:1788575559.810826:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9220. 00000100:00000001:0.0:1788575559.810828:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575559.810832:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4a80 x1875440516838144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810839:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575559.810842:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4a80 x1875440516838144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575584 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810849:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.810850:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575559.810853:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575559.810856:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575559.810859:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575559.810861:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310919849 00000100:00000001:0.0:1788575559.810863:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310923501 00000100:00000001:0.0:1788575559.810865:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310919849 00000100:00000001:0.0:1788575559.810867:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310923501 00000100:00000001:0.0:1788575559.810869:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575559.810877:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.810879:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575559.810883:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575559/real 1788575559] req@ffff8fc4ac9f4a80 x1875440516838144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575559.810891:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575559.810892:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575559.810894:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575559.810896:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575559.810898:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575559.810900:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575559.810903:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4a80 x1875440516838144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575559.810911:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4a80 x1875440516838144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810918:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575559.810919:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810921:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575559.810922:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575559.810925:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4a80 x1875440516838144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810932:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575559.810935:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4a80 x1875440516838144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810942:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575559.810944:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810945:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575559.810946:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810948:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575559.810950:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575559.810952:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575559.810958:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575559.810960:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575559.810965:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4a80 x1875440516838144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575559.810973:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516838144:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575559.810976:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.810978:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575559.810981:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4a80 x1875440516838144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575584 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575559.810987:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575559.810989:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575559.810990:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575559.810992:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575559.810994:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575559.810995:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575559.810997:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575559.810998:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788575559.811000:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575559.811001:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575559.811003:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4a80. 00000100:00000001:0.0:1788575559.811005:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575559.811006:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.811008:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575559.811011:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575559.811012:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575559.811014:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575559.811015:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575560.320933:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575560.320944:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575560.320948:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575560.320954:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575560.320959:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575560.320962:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575560.320980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575560.320982:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575560.320984:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575561.025018:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575561.025022:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575561.025025:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575561.344950:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575561.344956:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575561.344960:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575561.344966:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575561.344973:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575561.344978:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575561.344992:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575561.344996:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575561.345001:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575562.368964:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575562.368969:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575562.368971:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575562.368975:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575562.368978:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575562.368981:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575562.368985:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575562.368987:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575562.368991:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575562.369182:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575562.369185:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575562.369187:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575562.369190:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575562.369192:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575562.369195:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575562.369197:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575562.369199:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575562.369202:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575563.073004:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575563.073007:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575563.073009:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575563.392896:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575563.392904:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575563.392921:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575563.392927:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575563.392930:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575563.392935:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575563.392947:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575563.392949:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575563.392951:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575563.392965:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575563.392969:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575563.392970:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575563.392971:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575563.392972:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575563.392974:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575563.392976:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575563.392977:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575563.392979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575563.648896:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310923501, at 4310923520 00000100:00000010:0.0:1788575563.648913:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575564.097026:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575564.097029:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575564.097032:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575564.416925:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575564.416932:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575564.416935:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575564.416940:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575564.416946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575564.416951:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575564.416969:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575564.416973:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575564.416977:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.929237:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575564.929243:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575564.929251:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575564.929262:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516838272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575564.929280:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575564.929291:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575564.929295:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575564.929299:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575564.929303:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575564.929310:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516838272:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575564.929318:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575564.929322:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575564.929326:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575564.929330:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575564.929335:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575564.929339:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575564.929348:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575564.929353:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915108. 00000400:00000010:0.0:1788575564.929360:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed93b8. 00000100:00000200:0.0:1788575564.929368:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516838272, portal 25 00000100:00000001:0.0:1788575564.929372:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575564.929375:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788575564.929383:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516838272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575564.929396:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575564.929401:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575564.929405:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9220. 00000100:00000200:0.0:1788575564.929408:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516838272, offset 0 00000400:00000200:0.0:1788575564.929416:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575564.929425:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575564.929432:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575564.929439:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575564.929445:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575564.929451:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575564.929455:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575564.929462:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548201:548201:256:4294967295] 192.168.202.128@tcp LPNI seq info [548124:548124:8:4294967295] 00000400:00000200:0.0:1788575564.929470:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575564.929483:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575564.929498:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575564.929506:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56d00. 00000800:00000200:0.0:1788575564.929513:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788575564.929544:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575564.929551:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575564.929555:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310928621 00000100:00000010:0.0:1788575564.929561:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575564.929568:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575564.929574:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310928621 00000100:00000010:0.0:1788575564.929578:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50420. 00000100:00000001:0.0:1788575564.929582:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575564.929586:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575564.929588:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.929591:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.929594:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575564.929601:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575564.929604:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788575564.929607:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575564.929609:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575564.929611:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575564.929617:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516838400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575564.929630:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575564.929632:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575564.929635:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575564.929637:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575564.929640:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575564.929645:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516838400:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575564.929649:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575564.929653:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575564.929655:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575564.929658:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575564.929666:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575564.929669:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575564.929674:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575564.929677:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915e70. 00000400:00000010:0.0:1788575564.929682:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575564.929686:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516838400, portal 10 00000100:00000001:0.0:1788575564.929690:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575564.929693:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788575564.929703:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516838400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575564.929717:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575564.929723:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575564.929729:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9c38. 00000100:00000200:0.0:1788575564.929734:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516838400, offset 0 00000400:00000200:0.0:1788575564.929745:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575564.929751:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575564.929758:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575564.929763:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575564.929768:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575564.929772:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575564.929775:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575564.929778:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548202:548202:255:4294967295] 192.168.202.128@tcp LPNI seq info [548125:548125:7:4294967295] 00000400:00000200:0.0:1788575564.929785:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575564.929794:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575564.929803:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575564.929833:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56000. 00000800:00000200:0.0:1788575564.929838:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788575564.929846:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575564.929850:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575564.929856:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575564.929860:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310924819 00000100:00000001:0.0:1788575564.929872:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310928621 00000100:00000001:0.0:1788575564.929876:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310924819 00000100:00000001:0.0:1788575564.929881:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310928621 00000100:00000001:0.0:1788575564.929883:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575564.929888:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.929892:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.929897:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575564.929903:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575564.929906:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788575564.929910:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575564.929913:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575564.929915:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.929919:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.930176:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575564.930180:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575564.930184:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575564.930192:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516838528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575564.930206:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575564.930209:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575564.930213:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575564.930216:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575564.930220:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575564.930226:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516838528:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575564.930230:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575564.930233:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575564.930236:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575564.930238:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575564.930244:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575564.930248:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575564.930256:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575564.930260:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b1915b00. 00000400:00000010:0.0:1788575564.930263:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9b28. 00000100:00000200:0.0:1788575564.930273:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516838528, portal 10 00000100:00000001:0.0:1788575564.930276:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575564.930279:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788575564.930288:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516838528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575564.930300:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575564.930305:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575564.930309:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed92a8. 00000100:00000200:0.0:1788575564.930314:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516838528, offset 0 00000400:00000200:0.0:1788575564.930319:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575564.930327:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575564.930333:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575564.930337:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575564.930342:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575564.930346:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575564.930351:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575564.930354:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548203:548203:254:4294967295] 192.168.202.128@tcp LPNI seq info [548126:548126:6:4294967295] 00000400:00000200:0.0:1788575564.930362:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575564.930373:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575564.930380:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575564.930386:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56500. 00000800:00000200:0.0:1788575564.930395:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788575564.930406:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575564.930411:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575564.930417:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575564.930420:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310924826 00000100:00000001:0.0:1788575564.930426:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310928621 00000100:00000001:0.0:1788575564.930429:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310924826 00000100:00000001:0.0:1788575564.930433:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310928621 00000100:00000001:0.0:1788575564.930437:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575564.930451:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.930456:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575564.930461:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516838656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575564.930470:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575564.930471:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575564.930473:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575564.930475:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575564.930478:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575564.930481:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516838656:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575564.930484:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575564.930486:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575564.930488:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575564.930489:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575564.930493:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575564.930494:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575564.930498:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575564.930501:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19153c8. 00000400:00000010:0.0:1788575564.930508:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9bb0. 00000100:00000200:0.0:1788575564.930511:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516838656, portal 4 00000100:00000001:0.0:1788575564.930513:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575564.930515:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788575564.930520:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516838656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575564.930538:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575564.930541:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575564.930543:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9550. 00000100:00000200:0.0:1788575564.930545:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516838656, offset 0 00000400:00000200:0.0:1788575564.930548:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575564.930552:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575564.930555:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575564.930558:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575564.930567:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575564.930569:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575564.930571:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575564.930574:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548204:548204:253:4294967295] 192.168.202.128@tcp LPNI seq info [548127:548127:5:4294967295] 00000400:00000200:0.0:1788575564.930577:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575564.930583:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575564.930587:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575564.930591:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56c00. 00000800:00000200:0.0:1788575564.930595:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788575564.930602:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575564.930606:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575564.930608:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575564.930610:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310924966 00000100:00000001:0.0:1788575564.930612:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310928621 00000100:00000001:0.0:1788575564.930614:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310924966 00000100:00000001:0.0:1788575564.930615:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310928621 00000100:00000001:0.0:1788575564.930617:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575564.930620:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.930622:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.930624:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575564.930627:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575564.930629:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788575564.930631:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575564.930633:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575564.930634:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575564.930638:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516838784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575564.930644:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575564.930646:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575564.930647:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575564.930649:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575564.930655:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575564.930658:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516838784:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575564.930660:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575564.930663:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575564.930665:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575564.930666:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575564.930669:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575564.930670:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575564.930673:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575564.930675:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575564.930679:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575564.930681:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516838784, portal 4 00000100:00000001:0.0:1788575564.930683:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575564.930685:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788575564.930689:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516838784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575564.930695:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575564.930697:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575564.930699:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9880. 00000100:00000200:0.0:1788575564.930701:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516838784, offset 0 00000400:00000200:0.0:1788575564.930710:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575564.930715:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575564.930718:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575564.930721:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575564.930723:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575564.930725:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575564.930727:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575564.930730:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548205:548205:252:4294967295] 192.168.202.128@tcp LPNI seq info [548128:548128:4:4294967295] 00000400:00000200:0.0:1788575564.930734:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575564.930740:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575564.930744:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575564.930752:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56b00. 00000800:00000200:0.0:1788575564.930758:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000100:00000001:0.0:1788575564.930764:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575564.930766:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575564.930769:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575564.930771:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310924968 00000100:00000001:0.0:1788575564.930773:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310928621 00000100:00000001:0.0:1788575564.930775:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310924968 00000100:00000001:0.0:1788575564.930777:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310928621 00000100:00000001:0.0:1788575564.930778:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575564.930780:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.930783:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.930784:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575564.930787:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575564.930789:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788575564.930791:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575564.930792:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575564.930793:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575564.930795:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575565.441182:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575565.441187:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575565.441191:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575565.441197:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575565.441202:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575565.441206:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575565.441210:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575565.441212:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575565.441215:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575565.441321:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575565.441325:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575565.441329:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575565.441332:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575565.441334:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575565.441337:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575565.441355:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575565.441358:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575565.441361:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575566.144994:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575566.144997:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575566.144999:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575566.464943:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575566.464948:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575566.464952:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575566.464955:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575566.464959:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575566.464963:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575566.464973:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575566.464974:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575566.464976:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575567.489000:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575567.489005:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575567.489008:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575567.489011:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575567.489015:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575567.489019:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575567.489020:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575567.489022:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575567.489024:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575568.001707:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575568.001715:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575568.001724:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575568.001737:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575564/real 1788575568] req@ffff8fc4ac820e00 x1875440516838272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575568.001762:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575568.001766:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575568.001771:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915108. 00000400:00000200:0.0:1788575568.001779:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed93b8 00000400:00000010:0.0:1788575568.001783:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed93b8. 00000100:00000001:0.0:1788575568.001791:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575568.001801:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516838272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575568.001862:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516838272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.001870:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575568.001872:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.001877:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575568.001881:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575568.001885:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516838272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.001892:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575568.001898:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516838272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.001906:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575568.001910:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.001911:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575568.001912:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.001916:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575568.001919:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575568.001924:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575568.001930:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575568.001934:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575568.001939:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516838272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575568.001951:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516838272:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575568.001959:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575568.001964:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575568.001972:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516838272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002002:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575568.002008:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575568.002014:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575568.002022:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575568.002027:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575568.002032:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575568.002038:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575568.002043:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628c00. 02000000:00000001:0.0:1788575568.002052:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575568.002056:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575568.002059:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788575568.002068:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575568.002070:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002073:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575568.002075:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575568.002077:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575568.002080:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575568.002088:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575564/real 1788575568] req@ffff8fc4ac823800 x1875440516838528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575568.002118:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575568.002121:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575568.002125:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915b00. 00000400:00000200:0.0:1788575568.002132:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9b28 00000400:00000010:0.0:1788575568.002136:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9b28. 00000100:00000001:0.0:1788575568.002141:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575568.002149:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516838528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575568.002168:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516838528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002200:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575568.002204:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002207:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575568.002216:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575568.002222:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516838528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002229:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575568.002233:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516838528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002241:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575568.002243:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002245:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575568.002246:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002248:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575568.002253:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575568.002255:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575568.002261:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575568.002263:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575568.002267:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516838528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575568.002273:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516838528:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575568.002277:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575568.002279:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575564/real 1788575568] req@ffff8fc4ac820a80 x1875440516838656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575568.002286:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575568.002287:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575568.002290:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19153c8. 00000400:00000200:0.0:1788575568.002293:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9bb0 00000400:00000010:0.0:1788575568.002294:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9bb0. 00000100:00000001:0.0:1788575568.002298:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575568.002301:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516838656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575568.002314:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516838656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002321:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575568.002323:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002325:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575568.002326:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575568.002330:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516838656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002337:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575568.002340:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516838656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002347:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575568.002349:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002356:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575568.002358:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002360:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575568.002362:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575568.002365:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575568.002369:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575568.002372:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575568.002375:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516838656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575568.002383:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516838656:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575568.002387:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575568.002391:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575564/real 1788575568] req@ffff8fc4ac821880 x1875440516838400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575568.002409:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575568.002411:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575568.002414:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b1915e70. 00000400:00000200:0.0:1788575568.002417:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575568.002419:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575568.002421:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575568.002424:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516838400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575568.002431:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516838400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002437:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575568.002439:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002440:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575568.002442:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575568.002446:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516838400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002451:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575568.002454:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516838400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002460:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575568.002463:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002464:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575568.002465:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002467:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575568.002469:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575568.002473:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575568.002476:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575568.002479:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575568.002489:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516838400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575568.002496:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516838400:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575568.002500:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575568.002502:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575568.002506:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516838528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002513:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575568.002515:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575568.002517:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575568.002523:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575568.002524:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575568.002525:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575568.002527:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575568.002530:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788575568.002534:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575568.002537:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575568.002539:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788575568.002542:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575568.002543:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002545:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575568.002548:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516838656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002554:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575568.002556:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575568.002558:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575568.002561:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575568.002563:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575568.002564:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575568.002567:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575568.002568:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788575568.002586:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575568.002592:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575568.002594:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788575568.002597:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575568.002598:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002600:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575568.002604:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516838400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002614:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575568.002616:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575568.002618:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575568.002621:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575568.002622:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575568.002623:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575568.002626:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575568.002627:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788575568.002632:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575568.002633:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575568.002635:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788575568.002637:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575568.002639:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002641:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575568.002643:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575568.002645:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575568.002647:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575568.002651:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575564/real 1788575568] req@ffff8fc4ac821c00 x1875440516838784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575568.002661:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575568.002663:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575568.002665:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575568.002667:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575568.002669:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575568.002672:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575568.002675:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516838784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575568.002683:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516838784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002696:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575568.002698:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002700:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575568.002702:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575568.002706:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516838784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002712:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575568.002715:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516838784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002722:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575568.002725:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002726:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575568.002728:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002730:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575568.002733:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575568.002736:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575568.002740:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575568.002743:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575568.002746:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516838784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575568.002755:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516838784:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575568.002759:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002761:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575568.002764:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516838784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575589 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575568.002770:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575568.002771:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575568.002778:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575568.002781:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575568.002783:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575568.002784:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575568.002786:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575568.002788:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788575568.002790:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575568.002791:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575568.002792:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788575568.002795:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575568.002797:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002799:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.002846:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575568.002849:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575568.002852:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575568.002854:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575568.193101:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575568.193106:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575568.193110:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575568.512972:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575568.512981:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575568.512985:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575568.512995:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575568.513003:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575568.513011:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.513018:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575568.513021:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575568.513025:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575568.768943:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310928621, at 4310928640 00000100:00000010:0.0:1788575568.768963:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575569.217019:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575569.217026:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575569.217030:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575569.536894:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575569.536904:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575569.536906:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575569.536914:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575569.536938:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575569.536942:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575569.536950:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575569.536954:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575569.536958:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575569.537012:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575569.537016:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575569.537017:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575569.537020:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575569.537022:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575569.537024:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575569.537026:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575569.537027:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575569.537030:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575570.049238:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788575570.049248:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575570.049272:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575570.049279:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575570.049285:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575570.560951:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575570.560961:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575570.560964:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575570.560970:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575570.560976:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575570.560980:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575570.560983:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575570.560992:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575570.560998:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575570.561043:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575570.561047:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575570.561048:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575570.561051:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575570.561053:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575570.561056:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575570.561058:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575570.561066:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575570.561068:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575570.561079:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575570.561098:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575570.561100:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575570.561103:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575570.561105:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575570.561107:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575570.561316:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575570.561318:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575570.561319:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575571.265029:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575571.265032:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575571.265035:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575571.584967:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575571.584973:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575571.584979:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575571.584984:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575571.584990:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575571.584995:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575571.585017:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575571.585021:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575571.585026:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575571.585045:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575571.585050:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575571.585052:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575571.585055:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575571.585058:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575571.585061:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575571.585064:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575571.585066:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575571.585069:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575572.608957:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575572.608968:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575572.608973:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575572.608980:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575572.608986:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575572.608991:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575572.609008:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575572.609010:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575572.609012:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575572.609087:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575572.609090:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575572.609093:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575572.609098:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575572.609102:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575572.609105:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575572.609112:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575572.609115:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575572.609118:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575573.313057:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575573.313062:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575573.313065:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575573.632950:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575573.632961:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575573.632967:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575573.632976:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575573.632985:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575573.632991:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575573.632998:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575573.633001:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575573.633005:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575573.633031:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575573.633034:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575573.633036:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575573.633037:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575573.633039:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575573.633041:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575573.633042:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575573.633044:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575573.633046:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575573.888906:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310933742, at 4310933760 00000100:00000010:0.0:1788575573.888928:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfde0. 00000020:00000001:0.0:1788575574.336988:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575574.336991:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575574.336994:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575574.656952:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575574.656963:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575574.656997:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575574.657005:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575574.657011:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575574.657015:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575574.657028:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575574.657031:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575574.657034:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.169192:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575575.169201:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575575.169204:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575575.169216:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516839552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.169236:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575575.169240:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575575.169245:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575575.169250:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575575.169255:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575575.169262:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516839552:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575575.169276:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575575.169283:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575575.169289:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575575.169295:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575575.169304:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575575.169311:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575575.169322:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575575.169329:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575575.169338:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575575.169347:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516839552, portal 25 00000100:00000001:0.0:1788575575.169359:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575575.169362:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575575.169380:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516839552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.169409:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575575.169419:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575575.169446:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575575.169458:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516839552, offset 0 00000400:00000200:0.0:1788575575.169486:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575575.169506:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575575.169522:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575575.169538:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575575.169546:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575575.169552:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575575.169568:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575575.169580:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548211:548211:256:4294967295] 192.168.202.128@tcp LPNI seq info [548134:548134:8:4294967295] 00000400:00000200:0.0:1788575575.169584:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575575.169604:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575575.169617:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575575.169631:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575575.169642:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575575.169649:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575575.169674:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575575.169676:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575575.169697:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575575.169710:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575575.169719:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575575.169723:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575575.169731:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575575.169736:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516839552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.169766:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575575.169774:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516839552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.169837:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.169842:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575575.169856:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575575.169863:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575575.169866:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310938862 00000100:00000010:0.0:1788575575.169872:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575575.169881:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575575.169888:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310938862 00000100:00000010:0.0:1788575575.169892:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50a20. 00000100:00000001:0.0:1788575575.169897:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575575.169902:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575575.169907:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.169911:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575575.169919:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575575/real 1788575575] req@ffff8fc4901a4a80 x1875440516839552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575575.169941:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575575.169947:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575575.169949:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575575.169953:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575575.169955:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575575.169959:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575575.169964:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516839552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575575.169980:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516839552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.169996:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575575.170001:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.170005:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575575.170011:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575575.170024:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516839552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.170049:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575575.170057:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516839552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.170082:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575575.170087:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.170090:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575575.170092:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.170101:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575575.170103:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575575.170109:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575575.170114:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575575.170119:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575575.170124:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516839552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575575.170141:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516839552:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575575.170154:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.170158:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575575.170163:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516839552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.170177:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575575.170180:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575575.170184:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575575.170188:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575575.170191:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575575.170193:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575575.170195:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575575.170198:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575575.170204:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575575.170207:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575575.170211:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575575.170218:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575575.170221:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.170227:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.170243:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575575.170247:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.170252:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575575.170256:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575575.170259:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575575.170271:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516839680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.170290:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575575.170296:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575575.170301:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575575.170305:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575575.170312:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575575.170320:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516839680:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575575.170329:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575575.170334:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575575.170338:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575575.170343:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575575.170350:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575575.170357:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575575.170368:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575575.170373:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575575.170380:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575575.170386:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516839680, portal 10 00000100:00000001:0.0:1788575575.170391:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575575.170395:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575575.170409:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516839680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.170436:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575575.170446:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575575.170451:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575575.170456:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516839680, offset 0 00000400:00000200:0.0:1788575575.170468:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575575.170479:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575575.170492:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575575.170499:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575575.170507:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575575.170514:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575575.170520:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575575.170527:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548212:548212:256:4294967295] 192.168.202.128@tcp LPNI seq info [548135:548135:8:4294967295] 00000400:00000200:0.0:1788575575.170538:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575575.170554:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575575.170570:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575575.170580:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575575.170588:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575575.170595:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575575.170624:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575575.170628:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575575.170635:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575575.170645:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575575.170649:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575575.170653:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575575.170657:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575575.170664:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516839680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.170674:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575575.170679:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516839680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.170701:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.170703:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575575.170708:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575575.170713:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575575.170717:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575575.170719:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310935059 00000100:00000001:0.0:1788575575.170729:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310938862 00000100:00000001:0.0:1788575575.170731:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310935059 00000100:00000001:0.0:1788575575.170734:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310938862 00000100:00000001:0.0:1788575575.170736:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575575.170739:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.170741:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575575.170747:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575575/real 1788575575] req@ffff8fc4901a7100 x1875440516839680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575575.170756:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575575.170757:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575575.170759:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575575.170761:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575575.170763:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575575.170766:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575575.170769:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516839680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575575.170777:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516839680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.170783:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575575.170784:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.170785:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575575.170787:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575575.170790:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516839680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.170796:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575575.170799:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516839680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.170806:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575575.170829:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.170835:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575575.170837:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.170838:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575575.170841:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575575.170844:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575575.170848:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575575.170850:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575575.170854:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516839680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575575.170862:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516839680:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575575.170867:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575575.170871:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516839808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.170877:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575575.170878:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575575.170880:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575575.170883:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575575.170885:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575575.170888:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516839808:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575575.170891:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575575.170893:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575575.170895:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575575.170896:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575575.170900:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575575.170902:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575575.170906:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575575.170908:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575575.170912:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575575.170914:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516839808, portal 10 00000100:00000001:0.0:1788575575.170916:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575575.170921:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575575.170928:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516839808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.170935:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575575.170937:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575575.170939:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575575.170941:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516839808, offset 0 00000400:00000200:0.0:1788575575.170945:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575575.170949:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575575.170953:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575575.170956:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575575.170958:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575575.170960:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575575.170962:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575575.170965:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548213:548213:256:4294967295] 192.168.202.128@tcp LPNI seq info [548136:548136:8:4294967295] 00000400:00000200:0.0:1788575575.170969:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575575.170974:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575575.170978:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575575.170981:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575575.170984:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575575.170987:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575575.170997:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575575.170999:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575575.171001:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575575.171005:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575575.171008:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575575.171009:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575575.171012:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575575.171015:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516839808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171033:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575575.171037:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516839808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171043:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.171044:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575575.171047:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575575.171051:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575575.171054:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575575.171056:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310935067 00000100:00000001:0.0:1788575575.171058:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310938862 00000100:00000001:0.0:1788575575.171060:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310935067 00000100:00000001:0.0:1788575575.171062:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310938862 00000100:00000001:0.0:1788575575.171064:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575575.171066:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.171068:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575575.171071:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575575/real 1788575575] req@ffff8fc4901a5180 x1875440516839808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575575.171079:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575575.171080:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575575.171081:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575575.171083:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575575.171085:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575575.171088:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575575.171091:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516839808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575575.171099:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516839808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171105:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575575.171110:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171111:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575575.171113:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575575.171116:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516839808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171122:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575575.171126:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516839808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171132:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575575.171134:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171135:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575575.171136:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171138:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575575.171141:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575575.171143:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575575.171147:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575575.171149:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575575.171153:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516839808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575575.171160:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516839808:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575575.171164:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575575.171167:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516839936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171173:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575575.171174:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575575.171176:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575575.171178:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575575.171180:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575575.171186:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516839936:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575575.171189:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575575.171192:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575575.171193:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575575.171195:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575575.171198:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575575.171199:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575575.171203:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575575.171205:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575575.171208:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575575.171211:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516839936, portal 4 00000100:00000001:0.0:1788575575.171213:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575575.171214:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575575.171219:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516839936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171225:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575575.171227:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575575.171229:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575575.171231:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516839936, offset 0 00000400:00000200:0.0:1788575575.171234:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575575.171238:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575575.171241:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575575.171243:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575575.171246:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575575.171247:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575575.171249:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575575.171252:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548214:548214:256:4294967295] 192.168.202.128@tcp LPNI seq info [548137:548137:8:4294967295] 00000400:00000200:0.0:1788575575.171256:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575575.171261:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575575.171265:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575575.171268:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575575.171275:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575575.171278:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575575.171290:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575575.171292:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575575.171295:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575575.171298:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575575.171301:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575575.171303:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575575.171305:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575575.171308:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516839936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171316:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575575.171319:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516839936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171325:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.171327:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575575.171329:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575575.171332:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575575.171335:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575575.171337:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310935207 00000100:00000001:0.0:1788575575.171339:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310938862 00000100:00000001:0.0:1788575575.171341:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310935207 00000100:00000001:0.0:1788575575.171344:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310938862 00000100:00000001:0.0:1788575575.171345:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575575.171347:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.171348:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575575.171352:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575575/real 1788575575] req@ffff8fc4901a5c00 x1875440516839936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575575.171359:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575575.171364:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575575.171366:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575575.171368:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575575.171370:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575575.171373:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575575.171376:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516839936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575575.171383:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516839936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171389:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575575.171391:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171393:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575575.171394:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575575.171398:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516839936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171404:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575575.171407:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516839936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171413:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575575.171416:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171417:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575575.171418:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171420:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575575.171422:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575575.171424:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575575.171428:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575575.171430:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575575.171434:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516839936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575575.171451:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516839936:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575575.171455:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171456:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575575.171459:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516839680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171465:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575575.171467:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575575.171469:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575575.171472:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575575.171473:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575575.171475:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575575.171477:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575575.171479:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575575.171481:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575575.171482:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575575.171484:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575575.171488:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575575.171489:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171490:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575575.171493:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516839808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171499:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575575.171500:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575575.171502:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575575.171504:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575575.171505:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575575.171506:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575575.171508:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575575.171509:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575575.171511:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575575.171512:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575575.171513:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575575.171523:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575575.171524:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171526:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575575.171529:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516839936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171535:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575575.171536:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575575.171537:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575575.171540:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575575.171542:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575575.171543:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575575.171544:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575575.171546:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575575.171548:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575575.171549:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575575.171550:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575575.171552:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575575.171553:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171555:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171559:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575575.171560:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.171562:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575575.171564:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575575.171566:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575575.171569:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516840064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171576:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575575.171577:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575575.171579:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575575.171581:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575575.171583:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575575.171586:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516840064:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575575.171589:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575575.171591:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575575.171593:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575575.171598:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575575.171601:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575575.171602:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575575.171606:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575575.171608:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575575.171611:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575575.171614:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516840064, portal 4 00000100:00000001:0.0:1788575575.171616:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575575.171617:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575575.171622:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516840064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171628:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575575.171630:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575575.171632:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575575.171634:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516840064, offset 0 00000400:00000200:0.0:1788575575.171637:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575575.171640:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575575.171644:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575575.171646:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575575.171648:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575575.171650:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575575.171652:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575575.171655:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548215:548215:256:4294967295] 192.168.202.128@tcp LPNI seq info [548138:548138:8:4294967295] 00000400:00000200:0.0:1788575575.171658:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575575.171663:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575575.171667:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575575.171670:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575575.171673:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575575.171675:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575575.171697:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575575.171699:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575575.171706:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575575.171709:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575575.171712:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575575.171714:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575575.171716:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575575.171719:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516840064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171726:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575575.171728:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516840064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575600 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171734:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.171735:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575575.171738:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575575.171741:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575575.171743:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575575.171745:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310935209 00000100:00000001:0.0:1788575575.171747:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310938862 00000100:00000001:0.0:1788575575.171749:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310935209 00000100:00000001:0.0:1788575575.171751:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310938862 00000100:00000001:0.0:1788575575.171753:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575575.171755:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.171757:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575575.171760:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575575/real 1788575575] req@ffff8fc4901a7800 x1875440516840064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575575.171767:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575575.171768:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575575.171770:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575575.171772:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575575.171773:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575575.171775:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575575.171782:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516840064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575575.171790:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516840064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171796:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575575.171797:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171799:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575575.171800:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575575.171803:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516840064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171827:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575575.171831:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516840064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171837:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575575.171839:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171841:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575575.171842:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171844:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575575.171846:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575575.171848:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575575.171851:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575575.171853:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575575.171856:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516840064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575575.171862:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516840064:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575575.171865:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171867:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575575.171869:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516840064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575600 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575575.171879:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575575.171881:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575575.171882:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575575.171885:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575575.171886:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575575.171887:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575575.171889:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575575.171891:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575575.171893:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575575.171895:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575575.171896:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575575.171898:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575575.171899:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171902:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.171904:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575575.171906:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.171907:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575575.171909:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.680908:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575575.680917:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575575.680921:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575575.680927:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575575.680932:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575575.680937:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.680941:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575575.680945:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.680949:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575575.680996:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575575.680999:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575575.681001:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575575.681007:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575575.681011:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575575.681016:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.681019:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575575.681020:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575575.681028:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575576.385069:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575576.385074:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575576.385077:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575576.704900:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575576.704908:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575576.704910:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575576.704916:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575576.704920:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575576.704923:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575576.704925:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575576.704927:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575576.704930:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575576.705029:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575576.705033:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575576.705036:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575576.705043:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575576.705050:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575576.705054:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575576.705063:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575576.705066:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575576.705072:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575578.433038:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575578.433041:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575578.433044:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575578.752923:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575578.752928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575578.752931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575578.752937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575578.752942:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575578.752947:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575578.752964:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575578.752969:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575578.752973:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575579.008911:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310938862, at 4310938880 00000100:00000010:0.0:1788575579.008925:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575579.457006:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575579.457009:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575579.457028:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575579.778828:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575579.778833:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575579.778836:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575579.778842:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575579.778847:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575579.778852:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575579.778857:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575579.778861:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575579.778875:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.289083:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575580.289090:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575580.289095:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575580.289105:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516840192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.289121:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575580.289125:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575580.289132:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575580.289136:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575580.289143:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575580.289151:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516840192:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575580.289159:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575580.289165:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575580.289169:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575580.289176:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575580.289183:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575580.289189:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575580.289199:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575580.289206:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575580.289216:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575580.289226:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516840192, portal 25 00000100:00000001:0.0:1788575580.289233:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575580.289238:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575580.289250:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516840192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.289299:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575580.289307:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575580.289312:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575580.289317:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516840192, offset 0 00000400:00000200:0.0:1788575580.289328:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575580.289339:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575580.289349:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575580.289359:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575580.289367:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575580.289377:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575580.289384:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575580.289394:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548216:548216:256:4294967295] 192.168.202.128@tcp LPNI seq info [548139:548139:8:4294967295] 00000400:00000200:0.0:1788575580.289406:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575580.289426:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575580.289439:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575580.289448:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575580.289459:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575580.289466:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575580.289488:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575580.289493:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575580.289502:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575580.289512:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575580.289522:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575580.289527:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575580.289535:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575580.289545:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516840192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.289569:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575580.289577:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516840192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.289620:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.289625:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575580.289635:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575580.289643:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575580.289649:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310943981 00000100:00000010:0.0:1788575580.289655:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575580.289664:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575580.289674:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310943981 00000100:00000010:0.0:1788575580.289680:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b506c0. 00000100:00000001:0.0:1788575580.289686:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575580.289692:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575580.289697:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.289701:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575580.289707:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575580/real 1788575580] req@ffff8fc4901a5880 x1875440516840192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575580.289717:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575580.289719:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575580.289722:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575580.289726:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575580.289728:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575580.289732:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575580.289737:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516840192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575580.289747:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516840192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.289754:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575580.289757:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.289759:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575580.289762:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575580.289779:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516840192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.289792:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575580.289797:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516840192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.289804:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575580.289807:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.289849:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575580.289850:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.289852:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575580.289854:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575580.289857:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575580.289861:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575580.289864:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575580.289868:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516840192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575580.289876:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516840192:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575580.289881:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.289882:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575580.289885:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516840192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.289892:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575580.289894:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575580.289896:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575580.289899:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575580.289901:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575580.289903:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575580.289905:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575580.289907:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575580.289911:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575580.289918:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575580.289919:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575580.289925:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575580.289926:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.289928:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.289934:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575580.289936:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.289938:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575580.289940:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575580.289941:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575580.289946:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516840320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.289952:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575580.289954:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575580.289956:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575580.289958:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575580.289960:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575580.289962:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516840320:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575580.289965:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575580.289968:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575580.289969:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575580.289971:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575580.289974:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575580.289976:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575580.289979:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575580.289982:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575580.289986:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575580.289988:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516840320, portal 10 00000100:00000001:0.0:1788575580.289990:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575580.289992:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575580.289998:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516840320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290004:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575580.290006:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575580.290014:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575580.290016:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516840320, offset 0 00000400:00000200:0.0:1788575580.290020:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575580.290024:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575580.290028:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575580.290031:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575580.290033:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575580.290035:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575580.290038:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575580.290040:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548217:548217:256:4294967295] 192.168.202.128@tcp LPNI seq info [548140:548140:8:4294967295] 00000400:00000200:0.0:1788575580.290044:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575580.290050:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575580.290055:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575580.290058:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575580.290062:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575580.290064:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575580.290074:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575580.290076:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575580.290079:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575580.290082:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575580.290085:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575580.290087:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575580.290090:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575580.290094:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516840320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290101:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575580.290104:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516840320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290110:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.290112:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575580.290122:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575580.290125:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575580.290129:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575580.290131:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310940179 00000100:00000001:0.0:1788575580.290133:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310943981 00000100:00000001:0.0:1788575580.290135:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310940179 00000100:00000001:0.0:1788575580.290137:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310943981 00000100:00000001:0.0:1788575580.290139:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575580.290141:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.290143:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575580.290147:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575580/real 1788575580] req@ffff8fc4901a4700 x1875440516840320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575580.290155:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575580.290156:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575580.290158:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575580.290160:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575580.290162:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575580.290164:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575580.290167:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516840320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575580.290175:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516840320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290181:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575580.290183:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.290184:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575580.290186:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575580.290189:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516840320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290196:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575580.290199:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516840320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290212:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575580.290214:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.290216:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575580.290217:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.290219:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575580.290221:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575580.290223:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575580.290226:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575580.290228:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575580.290232:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516840320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575580.290240:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516840320:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575580.290243:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575580.290247:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516840448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290253:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575580.290254:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575580.290256:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575580.290257:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575580.290259:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575580.290262:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516840448:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575580.290266:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575580.290268:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575580.290269:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575580.290271:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575580.290274:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575580.290276:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575580.290285:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575580.290287:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575580.290290:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575580.290293:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516840448, portal 10 00000100:00000001:0.0:1788575580.290295:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575580.290296:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575580.290303:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516840448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290309:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575580.290311:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575580.290314:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575580.290316:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516840448, offset 0 00000400:00000200:0.0:1788575580.290321:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575580.290324:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575580.290328:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575580.290330:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575580.290333:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575580.290335:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575580.290338:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575580.290340:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548218:548218:256:4294967295] 192.168.202.128@tcp LPNI seq info [548141:548141:8:4294967295] 00000400:00000200:0.0:1788575580.290344:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575580.290350:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575580.290354:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575580.290357:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575580.290361:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575580.290364:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575580.290374:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575580.290376:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575580.290378:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575580.290381:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575580.290384:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575580.290391:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575580.290395:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575580.290399:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516840448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290406:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575580.290409:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516840448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290415:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.290416:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575580.290420:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575580.290422:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575580.290425:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575580.290427:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310940186 00000100:00000001:0.0:1788575580.290430:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310943981 00000100:00000001:0.0:1788575580.290431:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310940186 00000100:00000001:0.0:1788575580.290433:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310943981 00000100:00000001:0.0:1788575580.290435:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575580.290437:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.290439:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575580.290443:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575580/real 1788575580] req@ffff8fc4901a7480 x1875440516840448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575580.290450:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575580.290451:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575580.290453:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575580.290455:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575580.290457:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575580.290459:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575580.290462:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516840448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575580.290470:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516840448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290482:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575580.290483:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.290485:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575580.290486:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575580.290490:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516840448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290496:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575580.290500:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516840448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290506:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575580.290508:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.290510:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575580.290511:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.290513:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575580.290515:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575580.290517:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575580.290520:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575580.290522:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575580.290526:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516840448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575580.290534:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516840448:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575580.290538:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575580.290541:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516840576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290547:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575580.290548:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575580.290556:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575580.290558:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575580.290560:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575580.290563:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516840576:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575580.290567:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575580.290569:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575580.290570:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575580.290572:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575580.290575:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575580.290577:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575580.290580:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575580.290583:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575580.290587:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575580.290590:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516840576, portal 4 00000100:00000001:0.0:1788575580.290592:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575580.290593:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575580.290598:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516840576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290604:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575580.290606:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575580.290609:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575580.290611:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516840576, offset 0 00000400:00000200:0.0:1788575580.290615:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575580.290618:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575580.290622:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575580.290624:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575580.290626:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575580.290628:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575580.290630:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575580.290633:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548219:548219:256:4294967295] 192.168.202.128@tcp LPNI seq info [548142:548142:8:4294967295] 00000400:00000200:0.0:1788575580.290637:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575580.290642:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575580.290653:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575580.290656:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575580.290660:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575580.290663:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575580.290672:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575580.290674:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575580.290677:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575580.290680:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575580.290683:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575580.290685:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575580.290687:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575580.290691:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516840576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290698:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575580.290701:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516840576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290707:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.290709:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575580.290712:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575580.290715:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575580.290719:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575580.290721:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310940326 00000100:00000001:0.0:1788575580.290723:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310943981 00000100:00000001:0.0:1788575580.290725:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310940326 00000100:00000001:0.0:1788575580.290727:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310943981 00000100:00000001:0.0:1788575580.290729:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575580.290731:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.290733:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575580.290737:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575580/real 1788575580] req@ffff8fc4901a6300 x1875440516840576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575580.290751:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575580.290752:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575580.290754:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575580.290756:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575580.290758:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575580.290760:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575580.290779:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516840576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575580.290788:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516840576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290794:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575580.290795:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.290797:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575580.290798:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575580.290802:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516840576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290833:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575580.290837:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516840576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290844:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575580.290847:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.290848:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575580.290850:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.290852:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575580.290855:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575580.290857:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575580.290861:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575580.290864:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575580.290873:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516840576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575580.290881:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516840576:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575580.290884:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575580.290887:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516840704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290893:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575580.290894:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575580.290897:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575580.290898:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575580.290900:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575580.290903:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516840704:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575580.290906:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575580.290907:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575580.290909:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575580.290911:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575580.290914:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575580.290916:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575580.290919:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575580.290921:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575580.290925:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575580.290928:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516840704, portal 4 00000100:00000001:0.0:1788575580.290930:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575580.290931:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575580.290937:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516840704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.290943:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575580.290945:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575580.290947:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575580.290950:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516840704, offset 0 00000400:00000200:0.0:1788575580.290961:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575580.290964:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575580.290968:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575580.290971:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575580.290973:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575580.290975:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575580.290977:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575580.290980:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548220:548220:256:4294967295] 192.168.202.128@tcp LPNI seq info [548143:548143:8:4294967295] 00000400:00000200:0.0:1788575580.290984:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575580.290989:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575580.290993:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575580.290997:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575580.291000:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575580.291003:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1788575580.291012:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575580.291014:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575580.291017:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575580.291020:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575580.291023:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575580.291024:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575580.291027:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575580.291030:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516840704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.291038:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575580.291041:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516840704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575605 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.291047:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.291048:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575580.291051:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575580.291054:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575580.291057:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575580.291065:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310940329 00000100:00000001:0.0:1788575580.291068:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310943981 00000100:00000001:0.0:1788575580.291070:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310940329 00000100:00000001:0.0:1788575580.291071:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310943981 00000100:00000001:0.0:1788575580.291073:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575580.291075:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.291077:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575580.291081:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575580/real 1788575580] req@ffff8fc4901a4e00 x1875440516840704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575580.291089:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575580.291090:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575580.291092:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575580.291094:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575580.291096:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575580.291098:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575580.291101:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516840704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575580.291109:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516840704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.291115:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575580.291116:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.291118:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575580.291119:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575580.291122:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516840704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.291129:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575580.291132:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516840704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.291139:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575580.291146:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.291148:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575580.291149:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.291151:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575580.291153:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575580.291155:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575580.291159:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575580.291161:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575580.291165:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516840704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575580.291172:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516840704:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575580.291176:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.291178:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575580.291180:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516840320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.291186:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575580.291188:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575580.291190:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575580.291193:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575580.291194:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575580.291195:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575580.291197:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575580.291199:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575580.291202:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575580.291203:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575580.291204:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575580.291208:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575580.291209:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.291211:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575580.291214:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516840448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.291230:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575580.291232:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575580.291234:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575580.291236:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575580.291237:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575580.291238:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575580.291240:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575580.291241:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575580.291243:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575580.291245:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575580.291246:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575580.291249:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575580.291250:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.291252:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575580.291255:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516840576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.291262:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575580.291263:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575580.291264:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575580.291266:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575580.291267:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575580.291268:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575580.291270:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575580.291271:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575580.291273:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575580.291274:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575580.291276:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575580.291279:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575580.291280:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.291281:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575580.291285:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516840704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575605 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575580.291291:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575580.291292:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575580.291299:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575580.291301:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575580.291303:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575580.291304:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575580.291306:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575580.291307:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575580.291309:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575580.291310:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575580.291312:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575580.291315:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575580.291316:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.291319:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.291322:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575580.291324:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.291325:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575580.291327:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.800866:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575580.800873:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575580.800875:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575580.800879:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575580.800884:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575580.800889:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.800894:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575580.800898:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.800904:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575580.800939:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575580.800950:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575580.800957:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575580.800963:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575580.800968:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575580.800972:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.800980:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575580.800983:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575580.800986:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575581.505010:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575581.505013:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575581.505016:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575581.824909:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575581.824934:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575581.824938:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575581.824945:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575581.824951:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575581.824955:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575581.824968:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575581.824972:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575581.824975:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575582.848936:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575582.848946:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575582.848952:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575582.848968:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575582.848986:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575582.848997:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575582.849024:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575582.849027:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575582.849030:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575583.553059:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575583.553065:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575583.553069:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575583.873002:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575583.873014:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575583.873019:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575583.873035:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575583.873045:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575583.873051:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575583.873073:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575583.873076:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575583.873079:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575584.128968:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310943981, at 4310944000 00000100:00000010:0.0:1788575584.128994:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575584.577121:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575584.577125:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575584.577129:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575584.897290:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575584.897294:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575584.897295:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575584.897313:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575584.897316:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575584.897318:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575584.898107:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575584.898111:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575584.898115:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.409171:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575585.409178:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575585.409183:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575585.409194:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822300 x1875440516840832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.409213:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575585.409217:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575585.409221:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575585.409225:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575585.409229:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575585.409233:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516840832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575585.409238:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575585.409242:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575585.409246:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575585.409251:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575585.409258:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575585.409262:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575585.409270:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575585.409275:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575585.409283:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575585.409292:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516840832, portal 25 00000100:00000001:0.0:1788575585.409295:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575585.409297:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603412736 : -123400106138880 : ffff8fc4ac822300) 00000100:00000040:0.0:1788575585.409303:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822300 x1875440516840832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.409312:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575585.409318:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575585.409322:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575585.409335:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516840832, offset 0 00000400:00000200:0.0:1788575585.409345:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575585.409359:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575585.409371:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575585.409380:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575585.409389:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575585.409397:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575585.409402:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575585.409411:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548221:548221:256:4294967295] 192.168.202.128@tcp LPNI seq info [548144:548144:8:4294967295] 00000400:00000200:0.0:1788575585.409422:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575585.409437:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575585.409448:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575585.409456:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575585.409468:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575585.409476:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575585.409515:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575585.409521:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575585.409532:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575585.409543:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575585.409552:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575585.409557:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575585.409567:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575585.409576:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822300 x1875440516840832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.409598:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575585.409605:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822300 x1875440516840832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.409623:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.409626:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575585.409634:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575585.409642:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575585.409654:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310949101 00000100:00000010:0.0:1788575585.409661:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575585.409672:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575585.409679:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310949101 00000100:00000010:0.0:1788575585.409685:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b503c0. 00000100:00000001:0.0:1788575585.409692:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575585.409700:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575585.409704:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.409709:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575585.409717:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575585/real 1788575585] req@ffff8fc4ac822300 x1875440516840832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575585.409750:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575585.409754:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575585.409758:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575585.409764:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575585.409769:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575585.409773:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575585.409780:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822300 x1875440516840832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575585.409800:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822300 x1875440516840832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.409856:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575585.409860:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.409866:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575585.409871:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575585.409878:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822300 x1875440516840832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.409902:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575585.409912:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822300 x1875440516840832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.409951:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575585.409958:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.409962:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575585.409966:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.409971:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575585.409975:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575585.409982:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575585.409990:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575585.409996:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575585.410004:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822300 x1875440516840832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575585.410022:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516840832:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575585.410031:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410034:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575585.410040:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822300 x1875440516840832/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410051:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575585.410054:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575585.410057:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575585.410063:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575585.410065:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575585.410067:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575585.410072:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575585.410076:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a000. 02000000:00000001:0.0:1788575585.410084:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575585.410087:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575585.410091:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822300. 00000100:00000001:0.0:1788575585.410098:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575585.410101:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410105:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410121:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575585.410125:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.410129:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575585.410131:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575585.410133:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575585.410138:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820700 x1875440516840960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410142:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575585.410143:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575585.410145:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575585.410146:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575585.410148:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575585.410149:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516840960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575585.410152:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575585.410153:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575585.410154:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575585.410155:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575585.410158:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575585.410160:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575585.410165:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575585.410167:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575585.410171:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575585.410174:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516840960, portal 10 00000100:00000001:0.0:1788575585.410176:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575585.410177:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603405568 : -123400106146048 : ffff8fc4ac820700) 00000100:00000040:0.0:1788575585.410184:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820700 x1875440516840960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410190:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575585.410193:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575585.410195:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575585.410197:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516840960, offset 0 00000400:00000200:0.0:1788575585.410201:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575585.410205:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575585.410213:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575585.410215:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575585.410217:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575585.410218:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575585.410220:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575585.410222:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548222:548222:256:4294967295] 192.168.202.128@tcp LPNI seq info [548145:548145:8:4294967295] 00000400:00000200:0.0:1788575585.410224:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575585.410229:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575585.410232:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575585.410234:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575585.410237:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575585.410238:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575585.410246:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575585.410247:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575585.410249:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575585.410251:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575585.410253:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575585.410254:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575585.410257:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575585.410259:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820700 x1875440516840960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410264:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575585.410266:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820700 x1875440516840960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410271:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.410272:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575585.410274:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575585.410277:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575585.410279:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575585.410280:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310945299 00000100:00000001:0.0:1788575585.410285:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310949101 00000100:00000001:0.0:1788575585.410287:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310945299 00000100:00000001:0.0:1788575585.410288:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310949101 00000100:00000001:0.0:1788575585.410290:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575585.410291:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.410292:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575585.410294:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575585/real 1788575585] req@ffff8fc4ac820700 x1875440516840960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575585.410299:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575585.410300:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575585.410301:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575585.410302:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575585.410303:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575585.410305:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575585.410307:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820700 x1875440516840960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575585.410311:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820700 x1875440516840960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410315:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575585.410316:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410317:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575585.410318:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575585.410320:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820700 x1875440516840960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410323:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575585.410325:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820700 x1875440516840960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410329:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575585.410330:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410334:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575585.410335:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410336:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575585.410337:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575585.410339:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575585.410341:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575585.410342:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575585.410345:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820700 x1875440516840960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575585.410349:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516840960:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575585.410352:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575585.410354:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823480 x1875440516841088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410357:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575585.410358:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575585.410359:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575585.410360:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575585.410361:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575585.410363:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516841088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575585.410364:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575585.410365:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575585.410366:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575585.410367:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575585.410369:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575585.410371:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575585.410373:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575585.410374:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575585.410376:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575585.410378:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516841088, portal 10 00000100:00000001:0.0:1788575585.410379:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575585.410384:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603417216 : -123400106134400 : ffff8fc4ac823480) 00000100:00000040:0.0:1788575585.410388:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823480 x1875440516841088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410392:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575585.410394:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575585.410395:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575585.410397:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516841088, offset 0 00000400:00000200:0.0:1788575585.410399:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575585.410401:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575585.410403:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575585.410405:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575585.410406:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575585.410408:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575585.410409:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575585.410411:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548223:548223:256:4294967295] 192.168.202.128@tcp LPNI seq info [548146:548146:8:4294967295] 00000400:00000200:0.0:1788575585.410413:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575585.410417:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575585.410420:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575585.410421:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575585.410424:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575585.410426:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575585.410432:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575585.410433:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575585.410435:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575585.410437:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575585.410439:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575585.410440:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575585.410441:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575585.410443:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823480 x1875440516841088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410455:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575585.410457:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823480 x1875440516841088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410461:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.410462:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575585.410464:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575585.410466:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575585.410468:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575585.410470:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310945306 00000100:00000001:0.0:1788575585.410471:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310949101 00000100:00000001:0.0:1788575585.410473:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310945306 00000100:00000001:0.0:1788575585.410474:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310949101 00000100:00000001:0.0:1788575585.410475:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575585.410476:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.410477:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575585.410479:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575585/real 1788575585] req@ffff8fc4ac823480 x1875440516841088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575585.410483:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575585.410484:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575585.410485:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575585.410486:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575585.410487:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575585.410488:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575585.410490:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823480 x1875440516841088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575585.410494:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823480 x1875440516841088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410498:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575585.410501:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410502:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575585.410503:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575585.410506:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823480 x1875440516841088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410510:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575585.410512:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823480 x1875440516841088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410516:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575585.410518:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410518:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575585.410519:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410520:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575585.410522:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575585.410523:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575585.410525:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575585.410526:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575585.410528:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823480 x1875440516841088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575585.410534:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516841088:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575585.410537:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575585.410540:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821f80 x1875440516841216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410546:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575585.410548:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575585.410550:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575585.410551:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575585.410553:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575585.410561:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516841216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575585.410564:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575585.410566:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575585.410568:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575585.410570:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575585.410573:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575585.410574:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575585.410578:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575585.410580:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575585.410583:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575585.410585:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516841216, portal 4 00000100:00000001:0.0:1788575585.410586:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575585.410587:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603411840 : -123400106139776 : ffff8fc4ac821f80) 00000100:00000040:0.0:1788575585.410592:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821f80 x1875440516841216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410596:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575585.410597:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575585.410598:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575585.410600:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516841216, offset 0 00000400:00000200:0.0:1788575585.410602:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575585.410604:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575585.410606:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575585.410608:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575585.410610:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575585.410611:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575585.410613:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575585.410614:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548224:548224:256:4294967295] 192.168.202.128@tcp LPNI seq info [548147:548147:8:4294967295] 00000400:00000200:0.0:1788575585.410617:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575585.410621:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575585.410623:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575585.410625:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575585.410631:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575585.410633:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575585.410639:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575585.410641:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575585.410643:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575585.410645:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575585.410647:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575585.410648:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575585.410650:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575585.410652:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821f80 x1875440516841216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410657:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575585.410659:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821f80 x1875440516841216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410663:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.410664:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575585.410665:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575585.410667:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575585.410669:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575585.410671:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310945446 00000100:00000001:0.0:1788575585.410673:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310949101 00000100:00000001:0.0:1788575585.410675:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310945446 00000100:00000001:0.0:1788575585.410677:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310949101 00000100:00000001:0.0:1788575585.410680:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575585.410682:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.410683:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575585.410687:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575585/real 1788575585] req@ffff8fc4ac821f80 x1875440516841216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575585.410694:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575585.410700:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575585.410702:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575585.410704:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575585.410706:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575585.410709:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575585.410712:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821f80 x1875440516841216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575585.410720:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821f80 x1875440516841216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410726:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575585.410728:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410730:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575585.410731:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575585.410734:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821f80 x1875440516841216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410741:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575585.410745:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821f80 x1875440516841216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410752:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575585.410754:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410756:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575585.410757:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410758:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575585.410760:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575585.410762:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575585.410764:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575585.410766:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575585.410768:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821f80 x1875440516841216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575585.410781:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516841216:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575585.410784:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575585.410786:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac822d80 x1875440516841344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410792:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575585.410793:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575585.410795:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575585.410797:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575585.410799:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575585.410801:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516841344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575585.410804:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575585.410806:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575585.410824:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575585.410826:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575585.410830:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575585.410832:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575585.410835:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575585.410837:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575585.410840:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575585.410852:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516841344, portal 4 00000100:00000001:0.0:1788575585.410854:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575585.410855:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603415424 : -123400106136192 : ffff8fc4ac822d80) 00000100:00000040:0.0:1788575585.410861:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac822d80 x1875440516841344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410867:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575585.410870:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575585.410872:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575585.410875:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516841344, offset 0 00000400:00000200:0.0:1788575585.410879:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575585.410889:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575585.410893:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575585.410895:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575585.410897:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575585.410898:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575585.410900:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575585.410902:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548225:548225:256:4294967295] 192.168.202.128@tcp LPNI seq info [548148:548148:8:4294967295] 00000400:00000200:0.0:1788575585.410905:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575585.410909:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575585.410911:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575585.410914:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575585.410916:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575585.410918:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575585.410924:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575585.410926:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575585.410928:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575585.410930:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575585.410932:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575585.410933:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575585.410935:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575585.410937:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac822d80 x1875440516841344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410942:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575585.410944:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac822d80 x1875440516841344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575610 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410948:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.410949:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575585.410951:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575585.410953:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575585.410955:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575585.410959:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310945449 00000100:00000001:0.0:1788575585.410961:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310949101 00000100:00000001:0.0:1788575585.410963:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310945449 00000100:00000001:0.0:1788575585.410964:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310949101 00000100:00000001:0.0:1788575585.410965:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575585.410966:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.410967:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575585.410969:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575585/real 1788575585] req@ffff8fc4ac822d80 x1875440516841344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575585.410974:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575585.410975:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575585.410977:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575585.410978:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575585.410979:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575585.410980:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575585.410982:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac822d80 x1875440516841344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575585.410987:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac822d80 x1875440516841344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410990:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575585.410991:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.410992:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575585.410993:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575585.410995:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac822d80 x1875440516841344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.410998:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575585.411000:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac822d80 x1875440516841344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.411004:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575585.411010:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.411011:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575585.411012:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.411014:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575585.411016:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575585.411018:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575585.411022:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575585.411024:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575585.411027:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac822d80 x1875440516841344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575585.411036:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac822d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516841344:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575585.411039:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.411042:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575585.411044:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820700 x1875440516840960/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.411051:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575585.411052:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575585.411054:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575585.411056:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575585.411057:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575585.411057:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575585.411059:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575585.411060:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862bc00. 02000000:00000001:0.0:1788575585.411062:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575585.411063:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575585.411064:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820700. 00000100:00000001:0.0:1788575585.411066:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575585.411066:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.411068:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575585.411070:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823480 x1875440516841088/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.411081:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575585.411082:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575585.411083:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575585.411084:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575585.411085:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575585.411086:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575585.411087:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575585.411087:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b400. 02000000:00000001:0.0:1788575585.411088:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575585.411089:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575585.411090:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823480. 00000100:00000001:0.0:1788575585.411091:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575585.411091:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.411092:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575585.411094:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821f80 x1875440516841216/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.411097:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575585.411098:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575585.411099:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575585.411100:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575585.411100:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575585.411101:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575585.411102:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575585.411103:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629c00. 02000000:00000001:0.0:1788575585.411104:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575585.411105:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575585.411106:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821f80. 00000100:00000001:0.0:1788575585.411107:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575585.411108:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.411109:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575585.411110:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac822d80 x1875440516841344/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575610 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575585.411114:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575585.411114:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575585.411117:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575585.411119:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575585.411119:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575585.411120:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575585.411121:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575585.411122:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8628400. 02000000:00000001:0.0:1788575585.411124:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575585.411124:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575585.411125:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac822d80. 00000100:00000001:0.0:1788575585.411126:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575585.411127:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.411128:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575585.411130:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575585.411131:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575585.411132:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575585.411133:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575586.625109:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575586.625114:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575586.625118:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575586.944978:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575586.944984:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575586.944988:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575586.944995:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575586.945003:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575586.945009:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575586.945034:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575586.945037:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575586.945041:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575587.968977:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575587.968986:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575587.968991:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575587.968998:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575587.969007:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575587.969013:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575587.969032:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575587.969036:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575587.969040:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575587.969136:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575587.969157:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575587.969163:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575587.969169:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575587.969173:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575587.969180:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575587.969187:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575587.969193:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575587.969204:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575588.673046:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575588.673052:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575588.673055:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575588.992977:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575588.992989:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575588.992994:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575588.993003:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575588.993014:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575588.993024:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575588.993029:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575588.993032:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575588.993035:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575589.248943:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310949101, at 4310949120 00000100:00000010:0.0:1788575589.248956:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575589.697056:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575589.697061:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575589.697064:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575590.016984:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575590.016993:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575590.016997:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575590.017008:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575590.017025:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575590.017033:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575590.017040:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575590.017047:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575590.017051:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575590.528936:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575590.528947:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575590.528977:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575590.528980:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575590.528985:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575590.528992:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16277 00000400:00000200:0.0:1788575590.529001:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575590.529007:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575590.529010:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575590.529013:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529016:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575590.529018:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529020:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575590.529024:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575590.529026:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575590.529028:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575590.529031:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575590.529034:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575590.529036:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575590.529039:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529041:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575590.529043:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529045:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575590.529048:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575590.529051:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529053:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575590.529056:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575590.529059:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575590.529062:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.529074:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575590.529077:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575590.529088:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575590.529092:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575590.529103:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575590.529104:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.529107:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575590.529110:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575590.529114:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575590.529121:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575590.529127:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575590.529138:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516841472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575590.529148:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575590.529152:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575590.529157:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516841472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575590.529187:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575590.529192:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575590.529195:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575590.529197:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575590.529199:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575590.529201:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575590.529204:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16277 00000400:00000200:0.0:1788575590.529209:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575590.529212:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575590.529214:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575590.529217:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529219:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575590.529221:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529223:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575590.529225:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575590.529228:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575590.529231:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575590.529234:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575590.529243:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575590.529247:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575590.529254:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529259:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575590.529265:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529270:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575590.529276:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575590.529282:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529289:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575590.529293:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575590.529300:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575590.529306:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.529309:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575590.529315:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575590.529321:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.529323:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575590.529327:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575590.529329:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575590.529332:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575590.529333:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.529335:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575590.529336:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575590.529338:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575590.529341:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575590.529345:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575590.529350:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516841600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575590.529357:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575590.529360:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575590.529366:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516841600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575590.529394:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575590.529397:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575590.529401:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575590.529402:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575590.529404:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575590.529406:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575590.529409:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16277 00000400:00000200:0.0:1788575590.529415:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575590.529420:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575590.529423:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575590.529430:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529436:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575590.529442:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529447:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575590.529453:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575590.529456:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575590.529460:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575590.529464:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575590.529469:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575590.529474:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575590.529481:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529488:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575590.529493:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529498:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575590.529504:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575590.529511:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529517:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575590.529526:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575590.529530:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575590.529532:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.529534:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575590.529536:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575590.529538:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.529540:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575590.529544:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575590.529549:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575590.529556:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575590.529560:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.529566:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575590.529569:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575590.529571:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575590.529576:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575590.529580:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575590.529586:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516841728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575590.529594:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575590.529596:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575590.529604:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516841728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575590.529615:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575590.529621:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575590.529627:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575590.529632:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575590.529637:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575590.529644:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575590.529650:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16277 00000400:00000200:0.0:1788575590.529662:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575590.529668:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575590.529675:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575590.529689:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529693:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575590.529700:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529705:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575590.529709:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575590.529713:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575590.529717:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575590.529720:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575590.529723:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575590.529725:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575590.529729:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529731:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575590.529732:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529735:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575590.529737:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575590.529739:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575590.529742:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575590.529743:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575590.529759:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575590.529766:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.529770:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575590.529774:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575590.529779:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.529782:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575590.529788:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575590.529793:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575590.529800:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575590.529801:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.529804:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575590.529807:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575590.529836:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575590.529841:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575590.529865:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575590.529873:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516841856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575590.529902:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575590.529907:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575590.529916:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516841856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575590.529935:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575590.529941:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575590.529947:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575590.529951:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575590.529954:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575590.529961:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575590.529968:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16277 00000400:00000200:0.0:1788575590.529977:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575590.529985:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575590.529989:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575590.529996:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.530002:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575590.530008:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.530014:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575590.530019:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575590.530023:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575590.530027:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575590.530030:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575590.530035:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575590.530040:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575590.530045:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.530048:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575590.530060:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575590.530066:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575590.530072:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575590.530080:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575590.530088:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575590.530093:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575590.530100:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575590.530106:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.530111:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575590.530116:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575590.530122:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.530126:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575590.530132:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575590.530138:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575590.530144:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575590.530147:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575590.530152:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575590.530155:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575590.530159:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575590.530166:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575590.530173:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575590.530185:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516841984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575590.530201:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575590.530205:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575590.530211:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516841984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575590.530228:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575590.530235:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16288) 00000100:00000001:0.0:1788575591.040944:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575591.040950:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575591.040954:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575591.040977:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575591.040982:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575591.040986:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575591.040990:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575591.040993:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575591.040999:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575591.745026:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575591.745029:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575591.745031:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575592.064933:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575592.064938:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575592.064941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575592.064947:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575592.064953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575592.064958:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575592.064972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575592.064975:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575592.064978:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575593.089129:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575593.089132:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575593.089133:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575593.089136:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575593.089138:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575593.089140:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575593.089143:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575593.089144:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575593.089146:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575593.793049:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575593.793052:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575593.793055:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575594.112916:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575594.112923:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575594.112926:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575594.112931:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575594.112935:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575594.112941:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575594.112945:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575594.112948:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575594.112965:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575594.368935:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310954221, at 4310954240 00000100:00000010:0.0:1788575594.368952:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf1e0. 00000020:00000001:0.0:1788575594.817019:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575594.817023:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575594.817027:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575595.137104:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575595.137106:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575595.137107:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575595.137108:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575595.137109:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575595.137111:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575595.137112:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575595.137113:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575595.137114:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575596.160992:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575596.160998:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575596.161004:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575596.161014:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575596.161022:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575596.161032:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575596.161040:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575596.161048:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575596.161053:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575596.865137:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575596.865140:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575596.865143:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575597.185024:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575597.185037:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575597.185042:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575597.185049:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575597.185055:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575597.185063:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575597.185071:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575597.185080:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575597.185085:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575598.208984:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575598.209004:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575598.209008:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575598.209013:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575598.209019:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575598.209023:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575598.209028:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575598.209030:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575598.209033:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575598.913056:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575598.913060:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575598.913063:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575599.232938:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575599.232945:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575599.232951:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575599.232957:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575599.232966:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575599.232970:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575599.232988:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575599.232992:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575599.232995:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575599.233055:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575599.233059:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575599.233061:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575599.233065:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575599.233068:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575599.233071:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575599.233074:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575599.233075:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575599.233078:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575599.488879:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310959341, at 4310959360 00000100:00000010:0.0:1788575599.488893:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf5a0. 00000020:00000001:0.0:1788575599.937057:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575599.937060:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575599.937063:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575600.256945:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575600.256950:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575600.256953:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575600.256966:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575600.256973:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575600.256978:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575600.256985:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575600.256988:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575600.256992:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575601.280942:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575601.280948:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575601.280950:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575601.280961:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575601.280967:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575601.280972:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575601.280988:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575601.280992:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575601.280998:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575601.985065:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575601.985067:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575601.985069:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575602.304927:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575602.304936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575602.304940:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575602.304954:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575602.304961:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575602.304967:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575602.304973:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575602.304977:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575602.304983:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575602.305050:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575602.305054:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575602.305056:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575602.305059:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575602.305061:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575602.305063:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575602.305065:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575602.305066:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575602.305068:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575604.033071:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575604.033075:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575604.033092:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575604.608908:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310964461, at 4310964480 00000100:00000010:0.0:1788575604.608936:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfc60. 00000020:00000001:0.0:1788575605.057004:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575605.057008:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575605.057012:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575605.376952:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575605.376959:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575605.376963:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575605.376976:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575605.376981:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575605.376985:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575605.377004:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575605.377006:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575605.377009:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575606.400963:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575606.400970:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575606.400975:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575606.400983:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575606.400989:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575606.400995:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575606.401013:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575606.401016:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575606.401019:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575606.401114:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575606.401118:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575606.401119:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575606.401122:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575606.401125:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575606.401128:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575606.401131:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575606.401133:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575606.401135:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575607.105143:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575607.105151:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575607.105154:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575607.424979:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575607.425012:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575607.425016:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575607.425021:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575607.425027:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575607.425031:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575607.425056:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575607.425060:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575607.425063:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575608.448908:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575608.448916:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575608.448920:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575608.448930:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575608.448949:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575608.448956:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575608.448974:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575608.448977:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575608.448979:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575608.961238:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788575609.153222:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575609.153227:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575609.153230:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575609.472984:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575609.472997:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575609.473006:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575609.473016:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575609.473033:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575609.473043:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575609.473069:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575609.473073:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575609.473078:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575609.728910:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310969581, at 4310969600 00000100:00000010:0.0:1788575609.728928:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf780. 00000020:00000001:0.0:1788575610.177040:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575610.177043:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575610.177047:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575611.008965:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575611.008979:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575611.009001:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575611.009004:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575611.009009:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575611.009015:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16298 00000400:00000200:0.0:1788575611.009022:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575611.009027:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575611.009030:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575611.009032:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009035:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575611.009037:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009038:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575611.009041:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575611.009042:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575611.009044:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575611.009046:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575611.009048:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575611.009049:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575611.009051:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009052:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575611.009053:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009054:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575611.009056:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575611.009058:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009060:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575611.009062:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575611.009064:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575611.009066:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.009067:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575611.009069:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575611.009074:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575611.009076:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575611.009082:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575611.009083:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.009085:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575611.009087:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575611.009091:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575611.009097:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575611.009102:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575611.009108:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516844032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575611.009115:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575611.009118:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575611.009121:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516844032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575611.009136:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575611.009138:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575611.009140:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575611.009141:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575611.009142:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575611.009147:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575611.009149:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16298 00000400:00000200:0.0:1788575611.009152:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575611.009157:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575611.009159:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575611.009161:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009162:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575611.009164:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009165:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575611.009167:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575611.009168:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575611.009172:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575611.009173:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575611.009179:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575611.009182:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575611.009186:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009188:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575611.009192:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009195:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575611.009199:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575611.009203:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009207:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575611.009209:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575611.009215:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575611.009218:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.009221:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575611.009225:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575611.009230:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.009235:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575611.009241:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575611.009246:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575611.009251:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575611.009254:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.009258:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575611.009261:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575611.009265:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575611.009274:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575611.009282:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575611.009290:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516844160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575611.009311:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575611.009315:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575611.009324:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516844160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575611.009361:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575611.009366:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575611.009371:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575611.009375:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575611.009379:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575611.009384:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575611.009391:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16298 00000400:00000200:0.0:1788575611.009400:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575611.009407:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575611.009410:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575611.009416:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009419:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575611.009424:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009426:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575611.009428:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575611.009430:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575611.009432:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575611.009435:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575611.009437:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575611.009438:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575611.009444:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009446:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575611.009450:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009452:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575611.009454:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575611.009459:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009462:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575611.009470:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575611.009474:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575611.009479:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.009481:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575611.009483:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575611.009488:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.009493:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575611.009498:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575611.009507:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575611.009512:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575611.009514:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.009518:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575611.009521:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575611.009524:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575611.009530:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575611.009535:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575611.009544:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516844288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575611.009557:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575611.009560:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575611.009568:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516844288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575611.009581:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575611.009586:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575611.009592:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575611.009595:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575611.009599:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575611.009603:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575611.009608:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16298 00000400:00000200:0.0:1788575611.009627:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575611.009635:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575611.009640:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575611.009652:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009656:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575611.009658:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009663:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575611.009666:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575611.009668:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575611.009670:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575611.009672:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575611.009676:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575611.009678:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575611.009684:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009685:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575611.009690:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009693:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575611.009698:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575611.009703:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009707:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575611.009710:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575611.009714:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575611.009718:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.009721:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575611.009724:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575611.009729:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.009733:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575611.009737:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575611.009741:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575611.009746:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575611.009748:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.009750:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575611.009752:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575611.009754:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575611.009758:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575611.009767:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575611.009776:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516844416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575611.009790:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575611.009794:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575611.009801:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516844416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575611.009859:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575611.009863:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575611.009867:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575611.009869:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575611.009872:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575611.009876:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575611.009881:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16298 00000400:00000200:0.0:1788575611.009890:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575611.009896:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575611.009901:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575611.009907:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009911:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575611.009916:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009921:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575611.009925:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575611.009927:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575611.009931:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575611.009935:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575611.009939:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575611.009955:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575611.009961:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009965:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575611.009977:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009980:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575611.009983:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575611.009989:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575611.009993:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575611.009998:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575611.010003:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575611.010007:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.010011:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575611.010014:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575611.010019:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.010023:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575611.010028:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575611.010032:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575611.010036:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575611.010038:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575611.010041:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575611.010044:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575611.010048:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575611.010055:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575611.010062:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575611.010071:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516844544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575611.010086:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575611.010090:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575611.010098:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516844544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575611.010114:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575611.010120:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16308) 00000100:00000001:0.0:1788575611.521001:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575611.521009:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575611.521025:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575611.521048:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575611.521061:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575611.521065:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575611.521069:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575611.521075:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575611.521080:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575612.225067:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575612.225069:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575612.225071:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575612.544961:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575612.544968:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575612.544973:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575612.544983:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575612.544988:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575612.544995:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575612.545019:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575612.545027:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575612.545031:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575613.569047:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575613.569052:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575613.569054:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575613.569057:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575613.569060:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575613.569063:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575613.569066:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575613.569067:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575613.569070:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575613.569100:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575613.569106:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575613.569110:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575613.569115:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575613.569121:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575613.569126:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575613.569131:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575613.569135:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575613.569141:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575613.569179:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575613.569182:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575613.569197:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575613.569200:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575613.569204:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575613.569208:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575613.569212:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575613.569214:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575613.569217:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575614.273104:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575614.273108:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575614.273111:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575614.592923:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575614.592929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575614.592933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575614.592939:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575614.592945:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575614.592950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575614.592962:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575614.592965:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575614.592970:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575614.848883:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310974701, at 4310974720 00000100:00000010:0.0:1788575614.848899:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf120. 00000020:00000001:0.0:1788575615.296990:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575615.296993:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575615.296995:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575615.616936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575615.616956:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575615.616970:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575615.616975:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575615.616982:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575615.616992:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575615.617006:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575615.617010:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575615.617015:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.129237:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575616.129250:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575616.129257:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575616.129273:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516844672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.129319:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575616.129322:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575616.129334:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575616.129338:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575616.129342:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575616.129348:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516844672:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575616.129356:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575616.129364:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575616.129369:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575616.129375:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575616.129384:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575616.129393:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575616.129407:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575616.129417:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575616.129438:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575616.129450:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516844672, portal 25 00000100:00000001:0.0:1788575616.129460:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575616.129465:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575616.129481:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516844672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.129495:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575616.129506:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575616.129513:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575616.129520:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516844672, offset 0 00000400:00000200:0.0:1788575616.129538:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575616.129556:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575616.129569:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575616.129580:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575616.129590:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575616.129600:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575616.129606:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575616.129629:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548251:548251:256:4294967295] 192.168.202.128@tcp LPNI seq info [548174:548174:8:4294967295] 00000400:00000200:0.0:1788575616.129647:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575616.129670:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575616.129690:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575616.129700:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575616.129715:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575616.129722:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575616.129771:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575616.129782:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575616.129790:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575616.129805:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575616.129839:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575616.129843:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575616.129852:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575616.129861:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516844672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.129889:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575616.129899:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516844672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.129935:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.129940:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575616.129949:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575616.129957:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575616.129975:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4310979822 00000100:00000010:0.0:1788575616.129987:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575616.129998:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575616.130012:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4310979822 00000100:00000010:0.0:1788575616.130023:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b507e0. 00000100:00000001:0.0:1788575616.130031:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575616.130039:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575616.130054:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.130064:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575616.130074:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575616/real 1788575616] req@ffff8fc4901a4e00 x1875440516844672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575616.130107:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575616.130111:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575616.130117:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575616.130131:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575616.130136:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575616.130142:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575616.130150:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516844672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575616.130185:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516844672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.130206:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575616.130210:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.130219:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575616.130223:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575616.130233:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516844672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.130260:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575616.130274:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516844672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.130293:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575616.130305:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.130313:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575616.130317:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.130322:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575616.130327:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575616.130336:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575616.130347:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575616.130360:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575616.130373:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516844672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575616.130405:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516844672:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575616.130422:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.130429:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575616.130442:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516844672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.130470:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575616.130476:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575616.130482:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575616.130489:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575616.130495:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575616.130500:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575616.130506:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575616.130511:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575616.130519:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575616.130523:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575616.130529:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575616.130541:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575616.130544:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.130549:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.130561:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575616.130565:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.130571:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575616.130577:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575616.130581:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575616.130592:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516844800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.130658:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575616.130662:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575616.130670:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575616.130675:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575616.130686:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575616.130691:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516844800:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575616.130695:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575616.130698:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575616.130700:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575616.130702:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575616.130706:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575616.130709:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575616.130714:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575616.130717:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575616.130722:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575616.130726:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516844800, portal 10 00000100:00000001:0.0:1788575616.130729:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575616.130730:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575616.130739:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516844800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.130746:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575616.130752:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575616.130755:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575616.130759:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516844800, offset 0 00000400:00000200:0.0:1788575616.130765:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575616.130771:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575616.130775:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575616.130779:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575616.130782:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575616.130786:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575616.130789:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575616.130792:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548252:548252:256:4294967295] 192.168.202.128@tcp LPNI seq info [548175:548175:8:4294967295] 00000400:00000200:0.0:1788575616.130796:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575616.130806:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575616.130835:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575616.130843:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575616.130847:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575616.130851:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575616.130862:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575616.130863:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575616.130866:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575616.130870:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575616.130874:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575616.130875:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575616.130878:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575616.130882:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516844800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.130890:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575616.130893:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516844800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.130899:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.130900:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575616.130903:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575616.130908:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575616.130911:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575616.130913:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310976020 00000100:00000001:0.0:1788575616.130915:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310979822 00000100:00000001:0.0:1788575616.130917:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310976020 00000100:00000001:0.0:1788575616.130919:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310979822 00000100:00000001:0.0:1788575616.130922:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575616.130924:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.130925:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575616.130929:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575616/real 1788575616] req@ffff8fc4901a6300 x1875440516844800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575616.130948:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575616.130950:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575616.130952:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575616.130954:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575616.130955:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575616.130958:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575616.130973:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516844800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575616.130981:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516844800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.130987:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575616.130988:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.130990:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575616.130992:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575616.130995:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516844800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131001:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575616.131004:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516844800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131010:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575616.131013:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131014:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575616.131015:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131018:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575616.131020:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575616.131023:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575616.131026:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575616.131028:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575616.131037:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516844800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575616.131044:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516844800:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575616.131048:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575616.131051:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516844928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131057:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575616.131058:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575616.131060:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575616.131062:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575616.131064:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575616.131066:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516844928:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575616.131070:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575616.131072:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575616.131074:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575616.131075:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575616.131078:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575616.131080:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575616.131084:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575616.131086:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575616.131089:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575616.131091:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516844928, portal 10 00000100:00000001:0.0:1788575616.131093:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575616.131094:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575616.131100:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516844928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131106:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575616.131108:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575616.131110:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575616.131112:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516844928, offset 0 00000400:00000200:0.0:1788575616.131121:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575616.131125:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575616.131128:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575616.131131:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575616.131133:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575616.131135:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575616.131137:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575616.131140:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548253:548253:256:4294967295] 192.168.202.128@tcp LPNI seq info [548176:548176:8:4294967295] 00000400:00000200:0.0:1788575616.131144:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575616.131149:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575616.131154:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575616.131157:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575616.131160:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575616.131163:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575616.131172:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575616.131174:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575616.131177:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575616.131180:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575616.131183:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575616.131184:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575616.131187:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575616.131190:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516844928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131197:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575616.131200:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516844928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131205:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.131207:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575616.131209:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575616.131213:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575616.131216:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575616.131222:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310976027 00000100:00000001:0.0:1788575616.131224:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310979822 00000100:00000001:0.0:1788575616.131226:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310976027 00000100:00000001:0.0:1788575616.131228:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310979822 00000100:00000001:0.0:1788575616.131230:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575616.131232:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.131234:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575616.131237:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575616/real 1788575616] req@ffff8fc4901a7480 x1875440516844928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575616.131244:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575616.131245:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575616.131247:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575616.131249:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575616.131251:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575616.131254:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575616.131256:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516844928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575616.131263:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516844928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131270:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575616.131271:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131273:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575616.131274:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575616.131277:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516844928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131283:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575616.131287:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516844928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131293:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575616.131299:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131300:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575616.131301:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131303:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575616.131306:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575616.131308:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575616.131312:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575616.131313:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575616.131316:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516844928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575616.131324:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516844928:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575616.131327:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575616.131331:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516845056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131337:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575616.131339:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575616.131341:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575616.131343:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575616.131345:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575616.131348:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516845056:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575616.131351:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575616.131353:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575616.131355:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575616.131357:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575616.131359:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575616.131361:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575616.131365:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575616.131367:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575616.131370:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575616.131374:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516845056, portal 4 00000100:00000001:0.0:1788575616.131379:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575616.131381:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575616.131386:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516845056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131392:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575616.131394:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575616.131396:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575616.131398:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516845056, offset 0 00000400:00000200:0.0:1788575616.131402:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575616.131405:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575616.131409:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575616.131412:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575616.131414:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575616.131416:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575616.131418:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575616.131421:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548254:548254:256:4294967295] 192.168.202.128@tcp LPNI seq info [548177:548177:8:4294967295] 00000400:00000200:0.0:1788575616.131424:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575616.131430:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575616.131434:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575616.131437:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575616.131441:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575616.131444:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575616.131454:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575616.131456:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575616.131459:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575616.131463:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575616.131465:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575616.131467:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575616.131469:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575616.131473:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516845056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131490:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575616.131493:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516845056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131500:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.131501:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575616.131505:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575616.131508:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575616.131512:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575616.131513:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310976167 00000100:00000001:0.0:1788575616.131516:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310979822 00000100:00000001:0.0:1788575616.131518:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310976167 00000100:00000001:0.0:1788575616.131520:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310979822 00000100:00000001:0.0:1788575616.131522:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575616.131525:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.131526:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575616.131530:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575616/real 1788575616] req@ffff8fc4901a4700 x1875440516845056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575616.131537:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575616.131538:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575616.131540:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575616.131542:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575616.131544:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575616.131546:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575616.131549:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516845056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575616.131556:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516845056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131563:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575616.131570:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131572:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575616.131573:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575616.131578:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516845056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131584:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575616.131588:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516845056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131595:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575616.131598:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131599:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575616.131601:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131603:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575616.131605:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575616.131607:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575616.131610:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575616.131613:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575616.131617:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516845056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575616.131624:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516845056:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575616.131628:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131630:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575616.131633:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516844800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131640:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575616.131642:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575616.131644:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575616.131647:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575616.131648:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575616.131653:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575616.131655:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575616.131657:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575616.131660:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575616.131661:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575616.131663:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575616.131668:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575616.131669:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131671:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575616.131674:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516844928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131680:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575616.131681:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575616.131683:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575616.131685:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575616.131686:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575616.131687:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575616.131689:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575616.131690:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575616.131692:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575616.131693:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575616.131694:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575616.131697:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575616.131699:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131700:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575616.131703:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516845056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131709:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575616.131710:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575616.131711:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575616.131714:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575616.131715:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575616.131716:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575616.131718:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575616.131719:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575616.131721:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575616.131726:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575616.131728:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575616.131730:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575616.131731:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131733:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.131737:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575616.131738:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.131740:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575616.131742:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575616.131743:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575616.131747:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516845184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131753:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575616.131755:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575616.131757:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575616.131758:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575616.131760:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575616.131763:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516845184:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575616.131766:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575616.131768:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575616.131770:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575616.131771:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575616.131774:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575616.131776:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575616.131779:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575616.131781:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575616.131785:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575616.131788:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516845184, portal 4 00000100:00000001:0.0:1788575616.131789:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575616.131791:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575616.131796:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516845184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131802:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575616.131805:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575616.131830:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575616.131834:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516845184, offset 0 00000400:00000200:0.0:1788575616.131838:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575616.131842:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575616.131845:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575616.131848:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575616.131850:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575616.131852:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575616.131855:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575616.131858:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548255:548255:256:4294967295] 192.168.202.128@tcp LPNI seq info [548178:548178:8:4294967295] 00000400:00000200:0.0:1788575616.131861:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575616.131867:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575616.131872:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575616.131874:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575616.131878:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575616.131880:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575616.131892:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575616.131893:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575616.131895:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575616.131898:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575616.131901:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575616.131903:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575616.131905:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575616.131909:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516845184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131916:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575616.131918:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516845184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575641 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.131925:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.131926:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575616.131935:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575616.131938:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575616.131941:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575616.131943:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4310976170 00000100:00000001:0.0:1788575616.131945:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4310979822 00000100:00000001:0.0:1788575616.131948:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4310976170 00000100:00000001:0.0:1788575616.131950:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4310979822 00000100:00000001:0.0:1788575616.131952:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575616.131954:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.131956:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575616.131975:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575616/real 1788575616] req@ffff8fc4901a5880 x1875440516845184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575616.131983:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575616.131985:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575616.131987:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575616.131989:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575616.131991:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575616.131993:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575616.131995:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516845184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575616.132002:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516845184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.132008:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575616.132009:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.132011:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575616.132012:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575616.132015:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516845184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.132021:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575616.132025:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516845184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.132036:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575616.132038:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.132040:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575616.132041:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.132043:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575616.132045:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575616.132048:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575616.132052:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575616.132054:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575616.132058:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516845184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575616.132065:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516845184:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575616.132069:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.132071:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575616.132073:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516845184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575641 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575616.132079:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575616.132081:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575616.132083:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575616.132085:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575616.132086:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575616.132087:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575616.132089:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575616.132090:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575616.132093:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575616.132095:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575616.132096:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575616.132098:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575616.132100:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.132105:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.132108:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575616.132110:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.132111:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575616.132113:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.640927:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575616.640937:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575616.640949:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575616.640957:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575616.640975:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575616.640980:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.640986:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575616.640989:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.640993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575616.641062:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575616.641066:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575616.641067:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575616.641070:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575616.641074:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575616.641076:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.641079:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575616.641082:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575616.641084:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575617.345036:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575617.345039:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575617.345042:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575617.664962:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575617.664986:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575617.664990:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575617.664996:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575617.665014:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575617.665026:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575617.665054:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575617.665059:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575617.665068:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575618.688999:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575618.689006:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575618.689009:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575618.689046:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575618.689050:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575618.689052:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575618.689055:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575618.689056:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575618.689059:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575618.689292:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575618.689297:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575618.689299:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575618.689302:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575618.689305:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575618.689308:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575618.689310:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575618.689312:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575618.689314:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575619.393037:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575619.393041:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575619.393044:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575619.713031:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575619.713037:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575619.713040:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575619.713044:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575619.713047:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575619.713049:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575619.713052:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575619.713053:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575619.713056:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575619.968908:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310979822, at 4310979840 00000100:00000010:0.0:1788575619.968925:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575620.416966:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575620.416969:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575620.416994:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575620.736911:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575620.736920:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575620.736925:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575620.736942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575620.736946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575620.736964:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575620.736969:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575620.736985:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575620.736990:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575621.248929:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575621.248939:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575621.248942:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575621.248946:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575621.248951:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575621.248959:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16308 00000400:00000200:0.0:1788575621.248969:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575621.248988:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575621.248992:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575621.248996:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.248999:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575621.249002:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249005:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575621.249009:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575621.249011:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575621.249019:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575621.249023:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575621.249025:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575621.249027:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575621.249030:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249032:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575621.249034:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249036:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575621.249038:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575621.249041:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249044:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575621.249046:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575621.249064:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575621.249067:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.249069:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575621.249072:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575621.249078:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575621.249081:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575621.249085:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575621.249086:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.249090:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575621.249092:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575621.249096:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575621.249102:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575621.249109:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575621.249117:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516845312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575621.249127:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575621.249131:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575621.249136:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516845312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575621.249156:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575621.249160:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575621.249163:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575621.249164:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575621.249166:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575621.249169:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575621.249172:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16308 00000400:00000200:0.0:1788575621.249177:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575621.249181:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575621.249184:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575621.249189:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249192:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575621.249203:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249208:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575621.249214:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575621.249218:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575621.249222:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575621.249226:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575621.249232:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575621.249237:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575621.249245:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249251:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575621.249255:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249259:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575621.249264:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575621.249271:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249274:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575621.249279:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575621.249286:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575621.249292:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.249297:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575621.249301:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575621.249313:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.249317:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575621.249322:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575621.249326:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575621.249330:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575621.249333:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.249337:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575621.249339:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575621.249342:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575621.249349:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575621.249354:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575621.249365:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516845440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575621.249387:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575621.249391:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575621.249399:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516845440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575621.249411:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575621.249416:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575621.249423:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575621.249427:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575621.249429:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575621.249434:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575621.249439:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16308 00000400:00000200:0.0:1788575621.249448:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575621.249454:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575621.249458:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575621.249462:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249464:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575621.249467:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249471:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575621.249476:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575621.249478:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575621.249480:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575621.249484:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575621.249489:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575621.249492:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575621.249499:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249504:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575621.249509:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249513:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575621.249524:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575621.249530:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249534:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575621.249537:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575621.249542:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575621.249546:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.249550:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575621.249553:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575621.249557:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.249561:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575621.249568:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575621.249573:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575621.249580:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575621.249583:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.249587:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575621.249590:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575621.249594:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575621.249601:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575621.249607:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575621.249618:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516845568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575621.249635:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575621.249639:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575621.249650:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516845568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575621.249680:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575621.249685:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575621.249693:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575621.249697:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575621.249701:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575621.249707:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575621.249718:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16308 00000400:00000200:0.0:1788575621.249728:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575621.249743:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575621.249748:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575621.249754:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249758:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575621.249763:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249768:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575621.249772:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575621.249774:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575621.249777:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575621.249780:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575621.249785:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575621.249789:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575621.249794:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249799:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575621.249804:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249833:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575621.249838:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575621.249844:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575621.249849:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575621.249853:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575621.249858:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575621.249863:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.249867:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575621.249871:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575621.249877:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.249882:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575621.249889:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575621.249894:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575621.249899:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575621.249911:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.249915:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575621.249919:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575621.249923:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575621.249929:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575621.249936:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575621.249946:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516845696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575621.249963:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575621.249968:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575621.249988:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516845696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575621.250008:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575621.250012:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575621.250019:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575621.250021:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575621.250024:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575621.250028:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575621.250033:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16308 00000400:00000200:0.0:1788575621.250041:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575621.250049:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575621.250052:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575621.250059:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.250062:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575621.250068:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.250074:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575621.250078:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575621.250081:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575621.250084:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575621.250088:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575621.250093:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575621.250102:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575621.250109:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.250114:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575621.250118:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575621.250122:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575621.250127:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575621.250134:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575621.250139:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575621.250143:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575621.250147:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575621.250173:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.250178:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575621.250182:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575621.250187:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.250191:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575621.250197:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575621.250202:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575621.250206:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575621.250209:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575621.250221:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575621.250226:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575621.250229:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575621.250234:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575621.250238:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575621.250247:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516845824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575621.250261:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575621.250266:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575621.250275:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516845824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575621.250291:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575621.250302:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16318) 00000100:00000001:0.0:1788575621.760903:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575621.760909:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575621.760912:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575621.760919:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575621.760925:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575621.760931:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575621.760935:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575621.760940:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575621.760945:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575622.784911:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575622.784920:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575622.784925:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575622.784933:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575622.784941:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575622.784948:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575622.784964:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575622.784966:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575622.784969:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575622.785049:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575622.785052:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575622.785054:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575622.785056:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575622.785059:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575622.785061:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575622.785063:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575622.785065:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575622.785067:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575623.808923:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575623.808932:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575623.808936:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575623.808944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575623.808949:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575623.808956:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575623.808967:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575623.808971:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575623.808973:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575623.809015:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575623.809036:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575623.809037:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575623.809041:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575623.809043:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575623.809046:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575623.809048:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575623.809050:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575623.809056:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575624.513012:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575624.513014:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575624.513017:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575624.832930:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575624.832938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575624.832950:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575624.832959:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575624.832965:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575624.832970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575624.832975:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575624.832977:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575624.832979:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575624.833161:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575624.833163:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575624.833164:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575624.833165:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575624.833166:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575624.833168:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575624.833169:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575624.833170:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575624.833171:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575625.088920:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310984941, at 4310984960 00000100:00000010:0.0:1788575625.088938:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfb40. 00000020:00000001:0.0:1788575625.537039:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575625.537042:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575625.537045:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575625.856905:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575625.856915:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575625.856920:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575625.856940:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575625.856945:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575625.856949:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575625.856959:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575625.856962:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575625.856965:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575626.880914:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575626.880919:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575626.880923:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575626.880932:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575626.880939:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575626.880945:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575626.880951:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575626.880954:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575626.880959:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575626.880974:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575626.880979:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575626.880984:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575626.880989:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575626.881008:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575626.881011:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575626.881015:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575626.881017:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575626.881020:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575628.928943:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575628.928948:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575628.928952:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575628.928960:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575628.928966:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575628.928970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575628.928986:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575628.928989:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575628.928993:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575629.633101:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575629.633104:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575629.633106:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575629.952919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575629.952928:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575629.952955:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575629.952962:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575629.952967:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575629.952971:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575629.952984:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575629.952988:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575629.952991:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575630.208972:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310990061, at 4310990080 00000100:00000010:0.0:1788575630.208996:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cff00. 00000020:00000001:0.0:1788575630.656977:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575630.656980:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575630.656982:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575630.976920:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575630.976925:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575630.976930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575630.976937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575630.976941:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575630.976946:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575630.976951:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575630.976955:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575630.976968:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575630.976986:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575630.976989:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575630.976991:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575630.976992:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575630.976994:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575630.976996:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575630.976998:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575630.976999:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575630.977000:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575632.001095:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575632.001099:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575632.001101:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575632.001104:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575632.001107:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575632.001110:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575632.001112:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575632.001185:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575632.001188:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575632.005086:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575632.005091:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575632.005094:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575632.005316:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575632.005320:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575632.005322:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575632.005325:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575632.005326:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575632.005328:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575633.025085:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575633.025089:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575633.025090:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575633.025092:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575633.025095:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575633.025097:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575633.025099:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575633.025100:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575633.025102:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575634.049045:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575634.049061:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575634.049066:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575634.049077:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575634.049085:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575634.049093:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.049294:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575634.049298:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.049301:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575634.561295:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575634.561300:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575634.561308:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575634.561318:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575631/real 1788575634] req@ffff8fc4ac9f4700 x1875440516846592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575634.561346:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575634.561349:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575634.561355:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c45c60. 00000400:00000200:0.0:1788575634.561389:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fccc0 00000400:00000010:0.0:1788575634.561393:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fccc0. 00000100:00000001:0.0:1788575634.561400:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575634.561407:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4700 x1875440516846592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575634.561425:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4700 x1875440516846592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561442:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575634.561446:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561452:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575634.561456:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575634.561466:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4700 x1875440516846592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561479:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575634.561488:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4700 x1875440516846592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561503:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575634.561510:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561514:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575634.561517:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561522:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575634.561527:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575634.561533:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575634.561541:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575634.561546:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575634.561556:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4700 x1875440516846592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575634.561574:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516846592:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575634.561582:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575634.561591:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575634.561595:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4700 x1875440516846592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561601:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575634.561603:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575634.561607:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575634.561611:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575634.561614:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575634.561616:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575634.561619:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575634.561626:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37c00. 02000000:00000001:0.0:1788575634.561630:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575634.561632:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575634.561633:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4700. 00000100:00000001:0.0:1788575634.561637:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575634.561638:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561640:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575634.561642:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575634.561643:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575634.561645:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575634.561648:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575631/real 1788575634] req@ffff8fc4ac9f5f80 x1875440516846848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575634.561656:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575634.561657:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575634.561659:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c45aa8. 00000400:00000200:0.0:1788575634.561661:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc440 00000400:00000010:0.0:1788575634.561663:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc440. 00000100:00000001:0.0:1788575634.561665:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575634.561668:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5f80 x1875440516846848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575634.561674:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5f80 x1875440516846848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561680:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575634.561681:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561687:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575634.561689:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575634.561693:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5f80 x1875440516846848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561700:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575634.561713:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5f80 x1875440516846848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561719:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575634.561721:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561723:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575634.561724:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561725:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575634.561729:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575634.561731:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575634.561734:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575634.561737:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575634.561741:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5f80 x1875440516846848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575634.561749:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516846848:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575634.561753:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575634.561756:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575631/real 1788575634] req@ffff8fc4ac9f5180 x1875440516846976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575634.561764:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575634.561765:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575634.561767:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c45420. 00000400:00000200:0.0:1788575634.561770:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc088 00000400:00000010:0.0:1788575634.561772:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc088. 00000100:00000001:0.0:1788575634.561774:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575634.561782:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516846976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575634.561791:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516846976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561797:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575634.561799:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561801:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575634.561802:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575634.561805:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516846976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561847:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575634.561852:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516846976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561857:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575634.561859:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561861:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575634.561862:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561863:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575634.561865:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575634.561868:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575634.561871:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575634.561873:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575634.561876:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516846976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575634.561883:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516846976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575634.561886:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575634.561889:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575631/real 1788575634] req@ffff8fc4ac9f7800 x1875440516847104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575634.561906:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575634.561907:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575634.561909:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c45268. 00000400:00000200:0.0:1788575634.561911:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc908 00000400:00000010:0.0:1788575634.561912:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc908. 00000100:00000001:0.0:1788575634.561914:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575634.561917:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7800 x1875440516847104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575634.561924:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7800 x1875440516847104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561929:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575634.561930:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561932:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575634.561933:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575634.561936:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7800 x1875440516847104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561941:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575634.561945:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7800 x1875440516847104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.561950:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575634.561952:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561954:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575634.561955:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.561956:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575634.561958:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575634.561960:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575634.561963:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575634.561965:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575634.561974:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7800 x1875440516847104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575634.561981:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516847104:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575634.561985:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575634.561988:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575631/real 1788575634] req@ffff8fc4ac9f6a00 x1875440516846720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575634.561995:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575634.561997:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575634.561999:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4a7c452c0. 00000400:00000200:0.0:1788575634.562002:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc198 00000400:00000010:0.0:1788575634.562003:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc198. 00000100:00000001:0.0:1788575634.562006:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575634.562008:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516846720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575634.562029:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516846720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.562035:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575634.562036:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.562038:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575634.562039:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575634.562043:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516846720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.562049:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575634.562052:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516846720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.562059:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575634.562061:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.562068:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575634.562069:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.562071:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575634.562073:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575634.562076:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575634.562080:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575634.562087:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575634.562091:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516846720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575634.562098:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516846720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575634.562102:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.562104:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575634.562107:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5f80 x1875440516846848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.562113:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575634.562115:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575634.562117:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575634.562120:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575634.562122:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575634.562123:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575634.562125:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575634.562127:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788575634.562129:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575634.562131:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575634.562132:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5f80. 00000100:00000001:0.0:1788575634.562135:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575634.562137:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.562138:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575634.562142:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516846976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.562148:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575634.562158:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575634.562159:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575634.562162:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575634.562164:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575634.562165:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575634.562167:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575634.562169:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35400. 02000000:00000001:0.0:1788575634.562171:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575634.562172:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575634.562174:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788575634.562176:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575634.562177:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.562179:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575634.562183:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7800 x1875440516847104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.562189:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575634.562190:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575634.562192:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575634.562194:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575634.562195:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575634.562196:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575634.562198:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575634.562200:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788575634.562202:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575634.562203:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575634.562204:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7800. 00000100:00000001:0.0:1788575634.562206:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575634.562207:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.562209:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575634.562212:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516846720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575634.562218:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575634.562219:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575634.562220:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575634.562222:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575634.562223:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575634.562229:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575634.562231:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575634.562233:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788575634.562235:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575634.562237:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575634.562238:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788575634.562240:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575634.562241:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.562243:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575634.562251:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575634.562253:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575634.562255:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575634.562256:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575634.753088:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575634.753092:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575634.753095:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575635.072952:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575635.072959:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575635.072964:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575635.072971:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575635.072976:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575635.072980:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575635.072986:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575635.072990:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575635.072994:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575635.328913:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4310995181, at 4310995200 00000100:00000010:0.0:1788575635.328930:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf420. 00000020:00000001:0.0:1788575635.777051:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575635.777056:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575635.777058:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575636.096908:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575636.096915:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575636.096919:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575636.096927:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575636.096932:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575636.096938:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575636.096942:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575636.096958:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575636.096961:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.097039:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575636.097042:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575636.097043:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575636.097045:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575636.097047:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575636.097049:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575636.097051:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575636.097053:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575636.097054:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.097075:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575636.097077:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575636.097078:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575636.097079:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575636.097081:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575636.097083:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575636.097090:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575636.097092:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575636.097094:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575636.608964:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575636.608975:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575636.608977:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575636.608980:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575636.608984:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575636.608991:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16324 00000400:00000200:0.0:1788575636.609000:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575636.609006:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575636.609009:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575636.609012:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609016:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575636.609018:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609051:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575636.609056:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575636.609071:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575636.609074:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575636.609077:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575636.609080:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575636.609083:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575636.609088:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609091:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575636.609093:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609107:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575636.609110:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575636.609114:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609117:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575636.609121:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575636.609124:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575636.609126:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.609129:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575636.609131:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575636.609137:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575636.609140:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575636.609144:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575636.609146:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.609149:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575636.609152:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575636.609167:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575636.609175:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575636.609181:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575636.609191:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516847232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575636.609206:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575636.609221:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575636.609226:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516847232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575636.609261:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575636.609264:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575636.609268:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575636.609269:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575636.609271:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575636.609273:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575636.609276:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16324 00000400:00000200:0.0:1788575636.609280:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575636.609282:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575636.609285:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575636.609288:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609290:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575636.609292:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609294:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575636.609296:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575636.609298:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575636.609300:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575636.609303:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575636.609304:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575636.609307:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575636.609313:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609317:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575636.609320:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609323:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575636.609327:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575636.609332:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609337:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575636.609347:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575636.609355:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575636.609364:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.609369:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575636.609384:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575636.609389:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.609393:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575636.609400:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575636.609404:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575636.609407:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575636.609409:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.609413:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575636.609415:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575636.609417:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575636.609425:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575636.609430:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575636.609439:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516847360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575636.609452:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575636.609457:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575636.609466:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516847360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575636.609482:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575636.609488:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575636.609494:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575636.609495:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575636.609497:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575636.609501:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575636.609505:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16324 00000400:00000200:0.0:1788575636.609511:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575636.609525:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575636.609529:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575636.609541:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609545:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575636.609550:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609554:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575636.609558:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575636.609561:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575636.609564:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575636.609567:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575636.609570:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575636.609574:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575636.609579:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609583:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575636.609587:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609590:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575636.609591:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575636.609594:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609595:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575636.609597:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575636.609600:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575636.609602:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.609605:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575636.609607:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575636.609609:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.609610:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575636.609614:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575636.609616:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575636.609618:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575636.609620:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.609621:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575636.609623:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575636.609625:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575636.609628:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575636.609640:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575636.609645:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516847488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575636.609650:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575636.609651:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575636.609653:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516847488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575636.609659:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575636.609661:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575636.609663:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575636.609664:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575636.609665:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575636.609667:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575636.609668:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16324 00000400:00000200:0.0:1788575636.609940:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575636.609948:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575636.609951:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575636.609956:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609959:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575636.609964:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.609969:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575636.609973:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575636.609977:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575636.609980:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575636.609983:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575636.609989:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575636.609993:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575636.610001:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.610006:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575636.610029:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.610032:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575636.610034:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575636.610037:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575636.610039:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575636.610041:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575636.610044:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575636.610046:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.610048:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575636.610050:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575636.610052:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.610054:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575636.610062:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575636.610065:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575636.610067:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575636.610068:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.610070:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575636.610071:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575636.610076:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575636.610080:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575636.610083:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575636.610088:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516847616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575636.610094:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575636.610096:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575636.610103:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516847616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575636.610109:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575636.610112:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575636.610114:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575636.610116:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575636.610123:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575636.610126:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575636.610128:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16324 00000400:00000200:0.0:1788575636.610131:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575636.610134:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575636.610136:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575636.610138:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.610140:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575636.610142:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.610143:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575636.610145:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575636.610146:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575636.610148:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575636.610149:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575636.610151:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575636.610153:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575636.610155:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.610157:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575636.610158:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575636.610160:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575636.610161:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575636.610164:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575636.610166:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575636.610167:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575636.610170:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575636.610172:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.610173:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575636.610175:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575636.610177:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.610178:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575636.610185:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575636.610188:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575636.610190:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575636.610191:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575636.610193:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575636.610194:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575636.610196:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575636.610199:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575636.610202:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575636.610206:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516847744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575636.610212:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575636.610215:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575636.610219:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516847744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575636.610226:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575636.610230:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16334) 00000100:00000001:0.0:1788575637.120917:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575637.120924:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575637.120928:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575637.120933:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575637.120940:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575637.120951:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575637.120975:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575637.120978:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575637.120981:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575637.120997:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575637.121000:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575637.121002:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575637.121004:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575637.121007:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575637.121010:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575637.121012:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575637.121014:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575637.121016:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575638.144918:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575638.144925:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575638.144927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575638.144934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575638.144941:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575638.144947:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575638.145153:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575638.145155:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575638.145157:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575639.169025:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575639.169044:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575639.169049:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575639.169055:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575639.169058:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575639.169062:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575639.169065:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575639.169067:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575639.169070:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575639.170918:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575639.170923:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575639.170925:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575639.170928:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575639.170931:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575639.170933:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575639.170935:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575639.170937:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575639.170939:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575639.873009:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575639.873014:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575639.873016:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575640.193117:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575640.193122:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575640.193124:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575640.193127:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575640.193131:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575640.193135:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575640.193140:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575640.193142:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575640.193161:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575640.448904:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311000303, at 4311000320 00000100:00000010:0.0:1788575640.448918:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfea0. 00000020:00000001:0.0:1788575640.897008:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575640.897012:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575640.897015:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575641.216916:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575641.216923:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575641.216925:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575641.216931:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575641.216937:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575641.216941:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575641.216945:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575641.216949:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575641.216956:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575641.217042:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575641.217046:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575641.217048:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575641.217052:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575641.217054:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575641.217057:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575641.217060:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575641.217062:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575641.217065:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575642.240934:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575642.240943:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575642.240947:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575642.240956:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575642.240961:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575642.240966:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575642.240982:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575642.240985:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575642.240988:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575643.264905:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575643.264913:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575643.264918:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575643.264926:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575643.264946:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575643.264951:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575643.264961:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575643.264964:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575643.264967:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575644.288936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575644.288943:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575644.288945:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575644.288950:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575644.288956:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575644.288960:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575644.288963:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575644.288966:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575644.288969:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575644.993021:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575644.993025:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575644.993028:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575645.313071:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575645.313075:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575645.313077:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575645.313080:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575645.313084:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575645.313088:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575645.313092:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575645.313093:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575645.313095:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575645.568906:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311005421, at 4311005440 00000100:00000010:0.0:1788575645.568922:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf720. 00000020:00000001:0.0:1788575646.016998:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575646.017002:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575646.017005:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575646.848922:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575646.848929:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575646.848933:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575646.848938:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575646.848942:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575646.848963:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16334 00000400:00000200:0.0:1788575646.848972:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575646.848978:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575646.848981:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575646.848984:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.848987:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575646.848989:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.848991:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575646.848995:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575646.848997:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575646.849000:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575646.849003:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575646.849005:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575646.849008:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575646.849011:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849013:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575646.849015:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849017:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575646.849021:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575646.849024:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849027:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575646.849030:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575646.849034:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575646.849036:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849039:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575646.849041:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575646.849080:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575646.849084:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575646.849088:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575646.849090:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849092:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575646.849095:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575646.849105:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575646.849114:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575646.849122:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575646.849134:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516848512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575646.849145:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575646.849150:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575646.849155:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516848512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575646.849174:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575646.849177:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575646.849181:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575646.849182:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575646.849184:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575646.849187:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575646.849190:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16334 00000400:00000200:0.0:1788575646.849195:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575646.849199:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575646.849209:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575646.849216:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849220:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575646.849224:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849229:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575646.849233:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575646.849236:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575646.849239:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575646.849243:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575646.849248:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575646.849252:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575646.849259:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849270:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575646.849275:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849279:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575646.849284:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575646.849290:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849294:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575646.849298:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575646.849305:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575646.849310:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849314:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575646.849320:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575646.849333:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849336:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575646.849342:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575646.849346:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575646.849348:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575646.849350:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849354:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575646.849356:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575646.849358:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575646.849364:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575646.849368:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575646.849375:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516848640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575646.849384:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575646.849386:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575646.849391:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516848640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575646.849403:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575646.849408:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575646.849412:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575646.849420:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575646.849423:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575646.849427:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575646.849430:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16334 00000400:00000200:0.0:1788575646.849436:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575646.849442:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575646.849445:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575646.849450:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849453:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575646.849457:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849460:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575646.849462:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575646.849463:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575646.849464:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575646.849465:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575646.849468:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575646.849471:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575646.849475:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849477:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575646.849482:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849485:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575646.849487:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575646.849492:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849497:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575646.849498:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575646.849501:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575646.849505:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849506:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575646.849508:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575646.849511:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849518:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575646.849520:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575646.849526:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575646.849529:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575646.849531:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849534:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575646.849537:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575646.849538:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575646.849542:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575646.849546:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575646.849551:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516848768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575646.849559:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575646.849563:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575646.849566:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516848768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575646.849575:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575646.849578:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575646.849585:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575646.849586:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575646.849587:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575646.849589:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575646.849593:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16334 00000400:00000200:0.0:1788575646.849598:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575646.849602:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575646.849604:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575646.849607:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849610:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575646.849614:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849617:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575646.849620:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575646.849625:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575646.849626:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575646.849628:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575646.849629:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575646.849631:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575646.849634:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849638:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575646.849643:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849648:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575646.849653:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575646.849657:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849662:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575646.849666:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575646.849670:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575646.849673:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849677:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575646.849681:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575646.849683:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849686:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575646.849690:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575646.849694:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575646.849695:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575646.849697:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849698:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575646.849701:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575646.849703:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575646.849706:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575646.849711:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575646.849716:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516848896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575646.849725:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575646.849729:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575646.849738:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516848896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575646.849745:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575646.849749:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575646.849755:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575646.849758:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575646.849760:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575646.849764:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575646.849767:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16334 00000400:00000200:0.0:1788575646.849771:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575646.849774:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575646.849777:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575646.849781:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849784:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575646.849788:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849791:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575646.849801:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575646.849804:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575646.849836:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575646.849840:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575646.849845:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575646.849850:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575646.849859:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849865:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575646.849870:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849874:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575646.849878:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575646.849883:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575646.849889:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575646.849897:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575646.849901:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575646.849906:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849911:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575646.849914:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575646.849919:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849922:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575646.849929:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575646.849934:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575646.849939:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575646.849942:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575646.849947:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575646.849949:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575646.849953:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575646.849959:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575646.849966:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575646.849976:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516849024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575646.849989:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575646.849992:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575646.849996:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516849024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575646.850005:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575646.850008:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16344) 00000100:00000001:0.0:1788575648.384965:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575648.384970:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575648.384972:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575648.384975:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575648.384979:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575648.384981:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575648.384983:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575648.384985:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575648.384986:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575649.408919:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575649.408945:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575649.408948:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575649.408954:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575649.408961:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575649.408964:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575649.408967:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575649.408969:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575649.408973:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575650.112966:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575650.112968:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575650.112971:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575650.432918:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575650.432926:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575650.432931:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575650.432940:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575650.432955:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575650.432965:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575650.432982:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575650.432985:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575650.432988:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575650.688929:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311010541, at 4311010560 00000100:00000010:0.0:1788575650.688945:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfcc0. 00000020:00000001:0.0:1788575651.137059:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575651.137076:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575651.137079:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575651.968939:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575651.968949:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575651.968952:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575651.968956:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575651.968960:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575651.968968:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16339 00000400:00000200:0.0:1788575651.968977:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575651.968983:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575651.968986:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575651.968989:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969006:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575651.969007:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969009:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575651.969012:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575651.969013:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575651.969015:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575651.969017:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575651.969019:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575651.969020:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575651.969022:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969023:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575651.969024:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969025:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575651.969027:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575651.969029:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969030:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575651.969032:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575651.969034:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575651.969036:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969038:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575651.969039:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575651.969045:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575651.969047:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575651.969050:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575651.969051:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969053:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575651.969055:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575651.969058:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575651.969072:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575651.969079:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575651.969088:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516849152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575651.969110:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575651.969114:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575651.969119:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516849152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575651.969144:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575651.969147:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575651.969150:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575651.969152:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575651.969153:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575651.969155:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575651.969157:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16339 00000400:00000200:0.0:1788575651.969161:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575651.969164:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575651.969166:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575651.969169:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969171:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575651.969173:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969175:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575651.969177:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575651.969178:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575651.969180:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575651.969182:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575651.969183:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575651.969185:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575651.969188:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969190:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575651.969191:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969192:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575651.969198:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575651.969200:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969202:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575651.969203:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575651.969207:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575651.969209:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969210:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575651.969213:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575651.969215:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969216:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575651.969218:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575651.969220:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575651.969222:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575651.969222:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969224:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575651.969225:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575651.969226:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575651.969228:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575651.969231:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575651.969234:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516849280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575651.969239:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575651.969240:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575651.969242:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516849280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575651.969247:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575651.969248:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575651.969250:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575651.969251:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575651.969252:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575651.969254:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575651.969258:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16339 00000400:00000200:0.0:1788575651.969261:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575651.969263:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575651.969264:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575651.969266:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969267:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575651.969268:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969270:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575651.969271:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575651.969272:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575651.969273:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575651.969274:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575651.969275:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575651.969276:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575651.969277:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969278:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575651.969279:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969280:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575651.969281:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575651.969283:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969284:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575651.969285:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575651.969286:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575651.969287:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969288:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575651.969289:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575651.969291:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969292:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575651.969293:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575651.969294:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575651.969296:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575651.969299:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969300:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575651.969301:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575651.969302:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575651.969304:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575651.969306:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575651.969308:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516849408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575651.969312:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575651.969314:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575651.969316:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516849408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575651.969320:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575651.969321:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575651.969323:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575651.969324:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575651.969324:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575651.969326:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575651.969328:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16339 00000400:00000200:0.0:1788575651.969330:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575651.969331:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575651.969332:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575651.969334:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969335:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575651.969336:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969337:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575651.969338:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575651.969339:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575651.969340:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575651.969341:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575651.969342:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575651.969345:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575651.969347:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969349:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575651.969350:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969351:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575651.969352:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575651.969353:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969354:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575651.969355:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575651.969357:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575651.969358:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969359:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575651.969360:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575651.969361:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969362:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575651.969364:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575651.969365:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575651.969366:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575651.969366:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969367:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575651.969368:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575651.969369:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575651.969371:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575651.969372:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575651.969375:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516849536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575651.969379:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575651.969380:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575651.969382:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516849536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575651.969386:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575651.969389:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575651.969392:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575651.969392:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575651.969393:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575651.969395:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575651.969396:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16339 00000400:00000200:0.0:1788575651.969398:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575651.969400:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575651.969401:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575651.969402:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969403:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575651.969404:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969405:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575651.969406:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575651.969407:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575651.969408:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575651.969409:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575651.969411:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575651.969412:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575651.969413:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969414:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575651.969415:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969416:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575651.969417:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575651.969419:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575651.969420:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575651.969421:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575651.969422:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575651.969423:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969424:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575651.969427:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575651.969428:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969429:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575651.969431:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575651.969432:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575651.969434:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575651.969434:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575651.969435:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575651.969436:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575651.969437:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575651.969439:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575651.969441:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575651.969443:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516849664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575651.969447:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575651.969448:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575651.969450:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516849664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575651.969454:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575651.969456:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16349) 00000100:00000001:0.0:1788575652.480997:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575652.481000:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575652.481001:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575652.481003:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575652.481004:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575652.481006:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575652.481191:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575652.481193:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575652.481195:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575653.504905:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575653.504913:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575653.504916:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575653.504922:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575653.504929:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575653.504949:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575653.504954:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575653.504959:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575653.504963:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575654.528911:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575654.528916:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575654.528919:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575654.528925:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575654.528931:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575654.528934:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575654.528937:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575654.528941:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575654.528945:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575655.233111:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575655.233114:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575655.233116:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575655.552920:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575655.552929:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575655.552933:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575655.552940:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575655.552947:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575655.552953:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575655.553160:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575655.553162:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575655.553165:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575655.808920:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311015661, at 4311015680 00000100:00000010:0.0:1788575655.808934:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfa20. 00000020:00000001:0.0:1788575656.256999:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575656.257002:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575656.257004:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575656.576982:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575656.576988:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575656.576991:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575656.576996:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575656.577001:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575656.577006:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575656.577013:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575656.577024:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575656.577031:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575657.600915:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575657.600922:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575657.600926:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575657.600932:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575657.600937:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575657.600940:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575657.600954:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575657.600957:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575657.600959:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575657.601032:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575657.601036:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575657.601039:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575657.601042:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575657.601046:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575657.601049:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575657.601053:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575657.601054:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575657.601057:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575658.624962:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575658.624972:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575658.624974:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575658.624979:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575658.624983:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575658.624987:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575658.624991:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575658.624994:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575658.624998:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575659.648949:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575659.648958:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575659.648961:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575659.648968:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575659.648972:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575659.648976:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575659.648980:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575659.648983:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575659.648987:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575659.649090:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575659.649095:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575659.649098:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575659.649101:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575659.649104:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575659.649106:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575659.649108:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575659.649110:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575659.649112:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575660.353128:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575660.353131:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575660.353135:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575660.672929:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575660.672938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575660.672941:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575660.672947:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575660.672952:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575660.672957:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575660.672961:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575660.672963:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575660.672967:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575660.673041:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575660.673046:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575660.673047:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575660.673049:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575660.673051:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575660.673053:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575660.673345:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575660.673354:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575660.673356:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575660.928918:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311020782, at 4311020800 00000100:00000010:0.0:1788575660.928943:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf300. 00000020:00000001:0.0:1788575661.377298:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575661.377303:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575661.377306:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575661.696943:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575661.696952:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575661.696976:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575661.696987:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575661.696992:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575661.696998:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575661.697128:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575661.697132:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575661.697138:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575662.720963:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575662.720971:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575662.720978:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575662.720986:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575662.720993:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575662.721000:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575662.721007:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575662.721015:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575662.721019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575663.744924:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575663.744935:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575663.744939:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575663.744947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575663.744954:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575663.744961:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575663.744975:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575663.744983:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575663.744990:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575663.745045:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575663.745050:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575663.745052:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575663.745056:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575663.745059:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575663.745071:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575663.745074:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575663.745076:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575663.745079:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575664.768928:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575664.768934:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575664.768938:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575664.768944:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575664.768956:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575664.768961:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575664.768967:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575664.768971:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575664.768974:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575665.473020:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575665.473023:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575665.473026:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575665.793155:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575665.793159:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575665.793161:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575665.793164:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575665.793167:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575665.793170:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575665.793174:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575665.793178:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575665.793184:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575666.048922:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311025902, at 4311025920 00000100:00000010:0.0:1788575666.048940:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf840. 00000020:00000001:0.0:1788575666.497031:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575666.497035:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575666.497038:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575666.816880:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575666.816887:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575666.816890:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575666.816897:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575666.816902:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575666.816908:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575666.816924:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575666.816927:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575666.816930:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575667.841035:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575667.841040:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575667.841044:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575667.841047:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575667.841054:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575667.841082:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575667.841105:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575667.841107:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575667.841110:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575667.841142:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575667.841145:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575667.841146:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575667.841149:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575667.841152:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575667.841154:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575667.841156:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575667.841158:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575667.841160:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575668.864935:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575668.864942:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575668.864945:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575668.864949:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575668.864953:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575668.864958:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575668.864963:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575668.864966:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575668.864971:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575668.865206:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575668.865217:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575668.865219:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575668.865221:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575668.865223:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575668.865226:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575668.865227:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575668.865229:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575668.865231:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575669.888919:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575669.888926:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575669.888930:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575669.888937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575669.888943:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575669.888948:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575669.888956:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575669.888960:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575669.888979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575670.592992:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575670.592995:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575670.592998:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575670.912922:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575670.912929:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575670.912934:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575670.912942:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575670.912949:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575670.912954:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575670.912972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575670.912975:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575670.912978:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575671.168883:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311031021, at 4311031040 00000100:00000010:0.0:1788575671.168898:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf0c0. 00000020:00000001:0.0:1788575671.617098:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575671.617102:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575671.617105:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575671.937623:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575671.937629:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575671.937631:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575671.937634:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575671.937637:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575671.937642:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575671.937646:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575671.937652:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575671.937656:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575672.448955:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575672.448965:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575672.448968:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575672.448972:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575672.448978:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575672.448987:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16359 00000400:00000200:0.0:1788575672.448996:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575672.449002:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575672.449022:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575672.449026:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449029:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575672.449031:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449034:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575672.449038:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575672.449040:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575672.449042:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575672.449045:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575672.449047:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575672.449049:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575672.449052:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449054:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575672.449056:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449082:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575672.449084:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575672.449087:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449088:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575672.449091:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575672.449093:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575672.449095:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.449097:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575672.449098:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575672.449103:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575672.449106:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575672.449108:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575672.449110:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.449112:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575672.449114:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575672.449117:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575672.449122:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575672.449127:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575672.449137:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516851712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575672.449144:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575672.449148:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575672.449152:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516851712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575672.449176:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575672.449179:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575672.449183:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575672.449184:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575672.449186:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575672.449188:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575672.449191:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16359 00000400:00000200:0.0:1788575672.449196:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575672.449199:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575672.449202:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575672.449205:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449208:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575672.449211:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449216:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575672.449221:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575672.449223:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575672.449227:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575672.449230:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575672.449233:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575672.449238:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575672.449246:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449250:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575672.449254:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449265:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575672.449269:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575672.449275:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449278:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575672.449282:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575672.449288:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575672.449291:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.449293:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575672.449298:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575672.449301:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.449303:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575672.449307:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575672.449310:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575672.449313:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575672.449315:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.449319:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575672.449321:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575672.449324:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575672.449329:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575672.449335:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575672.449341:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516851840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575672.449353:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575672.449355:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575672.449357:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516851840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575672.449367:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575672.449372:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575672.449377:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575672.449379:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575672.449383:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575672.449395:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575672.449402:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16359 00000400:00000200:0.0:1788575672.449411:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575672.449418:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575672.449422:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575672.449429:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449434:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575672.449440:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449446:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575672.449451:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575672.449454:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575672.449458:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575672.449461:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575672.449465:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575672.449470:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575672.449477:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449480:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575672.449484:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449488:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575672.449492:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575672.449500:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449506:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575672.449510:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575672.449515:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575672.449520:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.449523:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575672.449527:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575672.449531:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.449532:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575672.449538:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575672.449548:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575672.449553:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575672.449556:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.449560:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575672.449564:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575672.449567:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575672.449573:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575672.449580:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575672.449589:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516851968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575672.449604:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575672.449608:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575672.449613:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516851968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575672.449631:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575672.449637:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575672.449643:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575672.449646:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575672.449649:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575672.449653:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575672.449659:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16359 00000400:00000200:0.0:1788575672.449665:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575672.449672:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575672.449675:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575672.449681:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449685:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575672.449689:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449693:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575672.449696:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575672.449699:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575672.449707:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575672.449711:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575672.449715:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575672.449719:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575672.449726:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449730:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575672.449735:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449740:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575672.449744:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575672.449750:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449754:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575672.449759:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575672.449762:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575672.449768:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.449770:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575672.449773:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575672.449777:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.449781:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575672.449785:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575672.449790:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575672.449793:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575672.449796:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.449799:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575672.449802:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575672.449804:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575672.449840:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575672.449844:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575672.449852:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516852096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575672.449862:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575672.449865:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575672.449873:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516852096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575672.449896:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575672.449901:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575672.449906:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575672.449910:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575672.449914:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575672.449919:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575672.449926:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16359 00000400:00000200:0.0:1788575672.449934:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575672.449939:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575672.449943:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575672.449949:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449954:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575672.449959:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449963:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575672.449965:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575672.449969:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575672.449973:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575672.449976:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575672.449981:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575672.449984:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575672.449992:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.449996:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575672.450001:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575672.450003:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575672.450005:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575672.450008:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575672.450010:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575672.450019:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575672.450023:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575672.450028:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.450032:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575672.450036:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575672.450038:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.450042:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575672.450047:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575672.450051:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575672.450053:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575672.450054:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575672.450056:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575672.450057:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575672.450070:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575672.450075:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575672.450079:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575672.450083:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516852224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575672.450088:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575672.450089:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575672.450091:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516852224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575672.450096:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575672.450099:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16370) 00000100:00000001:0.0:1788575672.960920:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575672.960931:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575672.960934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575672.960943:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575672.960950:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575672.960956:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575672.960973:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575672.960976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575672.960980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575673.984910:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575673.984930:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575673.984934:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575673.984940:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575673.984945:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575673.984950:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575673.984954:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575673.984957:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575673.984960:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575675.008916:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575675.008924:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575675.008929:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575675.008937:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575675.008943:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575675.008949:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575675.008964:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575675.008966:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575675.008969:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575675.008994:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575675.008997:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575675.008999:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575675.009002:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575675.009005:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575675.009008:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575675.009010:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575675.009019:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575675.009022:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575675.712988:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575675.712992:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575675.713003:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575676.032892:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575676.032899:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575676.032905:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575676.032913:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575676.032920:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575676.032928:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575676.032947:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575676.032953:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575676.032956:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575676.288936:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311036141, at 4311036160 00000100:00000010:0.0:1788575676.288952:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf660. 00000020:00000001:0.0:1788575676.737009:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575676.737012:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575676.737015:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575677.057176:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575677.057180:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575677.057181:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575677.057184:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575677.057186:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575677.057188:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575677.057190:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575677.057192:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575677.057194:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575679.105162:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575679.105167:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575679.105169:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575679.105172:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575679.105175:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575679.105178:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575679.105180:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575679.105182:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575679.105184:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575680.128947:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575680.128956:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575680.128961:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575680.128968:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575680.128974:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575680.128983:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575680.129023:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575680.129027:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575680.129032:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575680.833292:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575680.833296:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575680.833299:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575681.152934:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575681.152944:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575681.152972:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575681.152976:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575681.152981:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575681.152983:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575681.153017:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575681.153020:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575681.153025:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575681.408942:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311041261, at 4311041280 00000100:00000010:0.0:1788575681.408966:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf060. 00000020:00000001:0.0:1788575681.857118:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575681.857121:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575681.857124:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575682.177078:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575682.177081:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575682.177082:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575682.177085:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575682.177087:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575682.177090:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575682.177172:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575682.177175:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575682.177177:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575683.201104:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575683.201108:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575683.201110:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575683.201114:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575683.201117:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575683.201120:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575683.201123:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575683.201127:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575683.201130:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575684.737084:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788575684.737091:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788575684.737120:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788575684.737281:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575684.737295:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575684.737301:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575684.737318:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788575685.953049:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575685.953052:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575685.953069:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575686.272964:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575686.272968:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575686.272973:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575686.272978:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575686.272983:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575686.272988:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575686.273005:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575686.273008:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575686.273010:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575686.528931:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311046381, at 4311046400 00000100:00000010:0.0:1788575686.528948:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf240. 00000020:00000001:0.0:1788575686.977009:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575686.977013:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575686.977016:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575687.296929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575687.296939:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575687.296944:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575687.296952:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575687.296959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575687.296965:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.296981:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575687.296985:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.296989:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.809128:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575687.809134:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575687.809139:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575687.809149:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516853632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.809176:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575687.809180:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575687.809186:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575687.809189:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575687.809204:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575687.809212:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516853632:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575687.809220:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575687.809225:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575687.809228:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575687.809231:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575687.809236:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575687.809239:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575687.809248:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575687.809252:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575687.809259:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575687.809266:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516853632, portal 25 00000100:00000001:0.0:1788575687.809268:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575687.809270:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575687.809279:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516853632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.809289:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575687.809293:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575687.809296:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575687.809299:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516853632, offset 0 00000400:00000200:0.0:1788575687.809305:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575687.809315:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575687.809324:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575687.809329:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575687.809335:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575687.809340:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575687.809344:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575687.809349:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548321:548321:256:4294967295] 192.168.202.128@tcp LPNI seq info [548244:548244:8:4294967295] 00000400:00000200:0.0:1788575687.809355:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575687.809365:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575687.809375:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575687.809384:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575687.809394:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575687.809398:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575687.809416:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575687.809420:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575687.809427:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575687.809435:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575687.809439:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575687.809442:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575687.809448:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575687.809454:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516853632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.809465:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575687.809469:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516853632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.809476:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.809479:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575687.809485:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575687.809491:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575687.809497:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311051501 00000100:00000010:0.0:1788575687.809504:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575687.809511:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575687.809520:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311051501 00000100:00000010:0.0:1788575687.809526:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50ae0. 00000100:00000001:0.0:1788575687.809532:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575687.809538:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575687.809545:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.809549:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575687.809559:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575687/real 1788575687] req@ffff8fc4901a7800 x1875440516853632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575687.809599:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575687.809604:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575687.809607:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575687.809610:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575687.809613:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575687.809616:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575687.809622:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516853632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575687.809633:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516853632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.809648:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575687.809653:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.809658:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575687.809661:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575687.809668:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516853632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.809683:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575687.809691:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516853632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.809701:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575687.809705:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.809709:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575687.809711:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.809712:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575687.809716:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575687.809719:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575687.809725:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575687.809730:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575687.809736:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516853632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575687.809775:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516853632:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575687.809785:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.809791:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575687.809798:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516853632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.809822:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575687.809824:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575687.809825:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575687.809828:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575687.809830:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575687.809831:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575687.809834:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575687.809838:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575687.809844:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575687.809847:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575687.809851:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575687.809858:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575687.809862:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.809866:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.809877:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575687.809881:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.809887:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575687.809891:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575687.809894:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575687.809900:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516853760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.809922:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575687.809925:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575687.809931:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575687.809934:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575687.809940:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575687.809947:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516853760:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575687.809959:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575687.809962:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575687.809964:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575687.809967:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575687.809975:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575687.809978:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575687.809983:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575687.809986:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575687.809990:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575687.809993:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516853760, portal 10 00000100:00000001:0.0:1788575687.809995:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575687.809997:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575687.810004:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516853760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810010:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575687.810013:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575687.810015:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575687.810017:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516853760, offset 0 00000400:00000200:0.0:1788575687.810021:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575687.810025:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575687.810028:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575687.810031:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575687.810033:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575687.810036:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575687.810038:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575687.810040:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548322:548322:256:4294967295] 192.168.202.128@tcp LPNI seq info [548245:548245:8:4294967295] 00000400:00000200:0.0:1788575687.810053:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575687.810073:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575687.810078:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575687.810081:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575687.810086:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575687.810093:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575687.810103:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575687.810105:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575687.810108:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575687.810112:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575687.810116:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575687.810118:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575687.810120:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575687.810124:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516853760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810131:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575687.810135:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516853760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810141:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.810142:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575687.810145:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575687.810149:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575687.810153:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575687.810155:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311047699 00000100:00000001:0.0:1788575687.810158:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311051501 00000100:00000001:0.0:1788575687.810160:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311047699 00000100:00000001:0.0:1788575687.810162:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311051501 00000100:00000001:0.0:1788575687.810164:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575687.810166:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.810168:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575687.810172:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575687/real 1788575687] req@ffff8fc4901a5c00 x1875440516853760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575687.810180:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575687.810181:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575687.810187:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575687.810190:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575687.810192:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575687.810194:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575687.810198:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516853760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575687.810205:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516853760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810212:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575687.810213:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810215:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575687.810216:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575687.810219:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516853760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810225:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575687.810229:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516853760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810235:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575687.810237:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810239:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575687.810240:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810241:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575687.810243:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575687.810246:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575687.810249:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575687.810251:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575687.810254:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516853760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575687.810261:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516853760:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575687.810269:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575687.810272:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516853888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810278:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575687.810279:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575687.810281:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575687.810283:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575687.810285:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575687.810288:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516853888:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575687.810290:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575687.810292:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575687.810294:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575687.810295:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575687.810298:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575687.810300:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575687.810303:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575687.810305:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575687.810308:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575687.810311:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516853888, portal 10 00000100:00000001:0.0:1788575687.810313:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575687.810314:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575687.810319:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516853888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810324:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575687.810327:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575687.810329:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575687.810331:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516853888, offset 0 00000400:00000200:0.0:1788575687.810335:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575687.810339:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575687.810342:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575687.810345:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575687.810352:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575687.810354:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575687.810356:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575687.810359:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548323:548323:256:4294967295] 192.168.202.128@tcp LPNI seq info [548246:548246:8:4294967295] 00000400:00000200:0.0:1788575687.810363:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575687.810369:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575687.810373:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575687.810376:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575687.810379:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575687.810382:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575687.810391:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575687.810393:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575687.810396:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575687.810399:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575687.810402:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575687.810404:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575687.810406:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575687.810410:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516853888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810416:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575687.810418:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516853888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810422:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.810423:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575687.810425:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575687.810427:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575687.810429:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575687.810430:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311047706 00000100:00000001:0.0:1788575687.810432:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311051501 00000100:00000001:0.0:1788575687.810433:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311047706 00000100:00000001:0.0:1788575687.810438:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311051501 00000100:00000001:0.0:1788575687.810439:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575687.810441:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.810442:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575687.810445:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575687/real 1788575687] req@ffff8fc4901a5180 x1875440516853888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575687.810450:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575687.810451:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575687.810452:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575687.810453:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575687.810454:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575687.810455:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575687.810457:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516853888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575687.810462:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516853888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810465:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575687.810466:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810467:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575687.810468:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575687.810470:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516853888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810473:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575687.810475:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516853888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810479:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575687.810481:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810482:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575687.810482:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810486:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575687.810488:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575687.810489:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575687.810491:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575687.810492:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575687.810495:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516853888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575687.810499:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516853888:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575687.810501:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575687.810503:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516854016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810506:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575687.810507:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575687.810508:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575687.810509:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575687.810511:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575687.810512:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516854016:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575687.810514:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575687.810515:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575687.810516:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575687.810516:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575687.810518:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575687.810519:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575687.810521:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575687.810523:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575687.810525:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575687.810527:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516854016, portal 4 00000100:00000001:0.0:1788575687.810528:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575687.810529:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575687.810532:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516854016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810539:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575687.810541:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575687.810542:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575687.810544:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516854016, offset 0 00000400:00000200:0.0:1788575687.810546:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575687.810548:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575687.810551:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575687.810553:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575687.810554:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575687.810556:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575687.810557:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575687.810559:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548324:548324:256:4294967295] 192.168.202.128@tcp LPNI seq info [548247:548247:8:4294967295] 00000400:00000200:0.0:1788575687.810561:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575687.810565:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575687.810568:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575687.810569:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575687.810572:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575687.810574:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575687.810580:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575687.810581:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575687.810583:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575687.810585:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575687.810586:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575687.810587:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575687.810589:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575687.810591:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516854016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810595:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575687.810597:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516854016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810607:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.810608:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575687.810610:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575687.810612:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575687.810614:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575687.810615:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311047846 00000100:00000001:0.0:1788575687.810617:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311051501 00000100:00000001:0.0:1788575687.810618:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311047846 00000100:00000001:0.0:1788575687.810619:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311051501 00000100:00000001:0.0:1788575687.810620:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575687.810621:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.810622:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575687.810624:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575687/real 1788575687] req@ffff8fc4901a7100 x1875440516854016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575687.810628:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575687.810629:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575687.810630:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575687.810631:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575687.810632:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575687.810634:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575687.810635:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516854016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575687.810640:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516854016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810652:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575687.810654:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810655:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575687.810656:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575687.810658:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516854016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810665:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575687.810668:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516854016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810671:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575687.810672:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810673:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575687.810674:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810675:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575687.810676:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575687.810677:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575687.810679:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575687.810680:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575687.810682:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516854016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575687.810687:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516854016:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575687.810689:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810690:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575687.810691:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516853760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810695:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575687.810696:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575687.810697:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575687.810698:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575687.810699:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575687.810700:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575687.810701:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575687.810702:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575687.810706:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575687.810707:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575687.810708:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575687.810712:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575687.810712:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810713:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575687.810716:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516853888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810719:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575687.810720:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575687.810721:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575687.810722:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575687.810722:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575687.810723:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575687.810724:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575687.810725:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575687.810726:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575687.810726:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575687.810727:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575687.810728:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575687.810729:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810730:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575687.810731:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516854016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810734:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575687.810735:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575687.810736:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575687.810737:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575687.810738:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575687.810738:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575687.810739:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575687.810740:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575687.810741:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575687.810742:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575687.810743:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575687.810745:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575687.810749:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810750:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810754:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575687.810755:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.810757:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575687.810758:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575687.810759:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575687.810763:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516854144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810769:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575687.810771:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575687.810772:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575687.810774:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575687.810776:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575687.810779:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516854144:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575687.810782:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575687.810784:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575687.810785:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575687.810787:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575687.810790:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575687.810792:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575687.810795:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575687.810796:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575687.810798:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575687.810800:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516854144, portal 4 00000100:00000001:0.0:1788575687.810801:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575687.810802:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575687.810806:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516854144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810824:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575687.810825:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575687.810827:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575687.810828:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516854144, offset 0 00000400:00000200:0.0:1788575687.810831:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575687.810846:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575687.810848:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575687.810850:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575687.810852:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575687.810853:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575687.810854:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575687.810856:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548325:548325:256:4294967295] 192.168.202.128@tcp LPNI seq info [548248:548248:8:4294967295] 00000400:00000200:0.0:1788575687.810859:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575687.810862:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575687.810865:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575687.810867:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575687.810869:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575687.810871:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575687.810877:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575687.810878:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575687.810880:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575687.810882:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575687.810884:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575687.810885:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575687.810886:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575687.810889:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516854144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810893:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575687.810895:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516854144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575712 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810899:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.810900:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575687.810902:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575687.810903:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575687.810905:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575687.810911:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311047849 00000100:00000001:0.0:1788575687.810912:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311051501 00000100:00000001:0.0:1788575687.810913:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311047849 00000100:00000001:0.0:1788575687.810923:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311051501 00000100:00000001:0.0:1788575687.810924:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575687.810926:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.810927:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575687.810932:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575687/real 1788575687] req@ffff8fc4901a4a80 x1875440516854144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575687.810937:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575687.810938:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575687.810939:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575687.810941:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575687.810942:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575687.810943:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575687.810945:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516854144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575687.810949:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516854144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810953:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575687.810954:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810955:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575687.810955:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575687.810957:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516854144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810961:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575687.810963:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516854144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.810966:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575687.810970:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810971:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575687.810972:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810973:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575687.810975:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575687.810976:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575687.810980:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575687.810982:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575687.810985:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516854144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575687.810993:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516854144:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575687.810996:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.810998:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575687.811001:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516854144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575712 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575687.811006:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575687.811007:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575687.811009:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575687.811012:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575687.811013:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575687.811014:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575687.811016:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575687.811018:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575687.811021:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575687.811022:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575687.811024:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575687.811027:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575687.811028:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.811030:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575687.811033:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575687.811034:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575687.811036:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575687.811041:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575688.320930:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575688.320938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575688.320941:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575688.320948:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575688.320961:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575688.320966:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575688.322143:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575688.322146:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575688.322157:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575689.344919:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575689.344927:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575689.344931:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575689.344938:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575689.344944:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575689.344950:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575689.344959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575689.344964:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575689.344971:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575690.368920:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575690.368926:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575690.368930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575690.368936:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575690.368942:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575690.368948:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575690.368958:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575690.368962:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575690.368965:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575691.073018:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575691.073022:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575691.073026:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575691.392931:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575691.392936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575691.392939:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575691.392943:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575691.392948:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575691.392953:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575691.392972:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575691.392980:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575691.392983:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575691.648922:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311051501, at 4311051520 00000100:00000010:0.0:1788575691.648939:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575692.097011:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575692.097015:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575692.097018:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575692.417022:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575692.417026:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575692.417028:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575692.417031:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575692.417034:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575692.417036:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575692.417038:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575692.417040:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575692.417043:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575692.928934:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575692.928956:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575692.928960:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575692.928965:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575692.928971:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575692.928980:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16380 00000400:00000200:0.0:1788575692.928990:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575692.929004:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575692.929007:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575692.929010:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929014:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575692.929017:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929020:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575692.929023:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575692.929025:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575692.929028:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575692.929030:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575692.929048:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575692.929051:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575692.929064:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929066:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575692.929068:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929070:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575692.929072:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575692.929075:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929078:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575692.929081:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575692.929084:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575692.929087:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.929089:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575692.929091:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575692.929097:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575692.929100:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575692.929104:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575692.929106:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.929108:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575692.929111:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575692.929114:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575692.929119:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575692.929125:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575692.929131:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516854272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575692.929138:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575692.929141:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575692.929144:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516854272/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575692.929162:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575692.929164:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575692.929172:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575692.929173:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575692.929175:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575692.929178:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575692.929181:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16380 00000400:00000200:0.0:1788575692.929186:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575692.929191:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575692.929194:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575692.929197:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929201:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575692.929213:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929222:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575692.929227:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575692.929230:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575692.929234:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575692.929237:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575692.929244:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575692.929252:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575692.929264:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929270:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575692.929277:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929282:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575692.929288:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575692.929296:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929303:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575692.929310:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575692.929319:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575692.929324:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.929329:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575692.929335:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575692.929345:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.929354:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575692.929364:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575692.929366:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575692.929368:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575692.929369:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.929370:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575692.929373:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575692.929375:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575692.929378:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575692.929386:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575692.929396:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516854400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575692.929412:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575692.929414:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575692.929417:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516854400/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575692.929426:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575692.929429:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575692.929440:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575692.929442:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575692.929449:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575692.929455:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575692.929460:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16380 00000400:00000200:0.0:1788575692.929466:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575692.929470:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575692.929477:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575692.929485:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929491:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575692.929495:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929506:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575692.929512:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575692.929513:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575692.929518:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575692.929521:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575692.929528:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575692.929532:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575692.929545:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929550:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575692.929557:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929559:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575692.929562:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575692.929567:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929569:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575692.929578:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575692.929582:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575692.929585:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.929586:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575692.929588:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575692.929593:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.929595:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575692.929604:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575692.929606:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575692.929608:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575692.929609:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.929611:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575692.929613:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575692.929616:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575692.929624:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575692.929628:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575692.929634:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516854528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575692.929658:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575692.929662:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575692.929672:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516854528/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575692.929703:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575692.929713:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575692.929718:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575692.929721:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575692.929723:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575692.929733:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575692.929741:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16380 00000400:00000200:0.0:1788575692.929757:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575692.929770:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575692.929779:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575692.929785:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929790:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575692.929798:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929803:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575692.929851:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575692.929854:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575692.929857:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575692.929862:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575692.929870:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575692.929880:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575692.929890:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929897:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575692.929903:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929909:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575692.929926:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575692.929934:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575692.929942:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575692.929947:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575692.929951:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575692.929958:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.929960:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575692.929962:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575692.929964:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.929967:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575692.929972:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575692.929980:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575692.929989:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575692.929992:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.929998:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575692.930002:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575692.930006:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575692.930013:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575692.930019:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575692.930032:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516854656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575692.930048:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575692.930051:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575692.930074:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516854656/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575692.930094:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575692.930097:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575692.930108:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575692.930112:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575692.930116:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575692.930124:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575692.930140:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16380 00000400:00000200:0.0:1788575692.930149:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575692.930161:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575692.930165:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575692.930174:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.930181:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575692.930187:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.930195:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575692.930202:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575692.930206:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575692.930209:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575692.930214:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575692.930222:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575692.930230:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575692.930240:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.930248:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575692.930254:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575692.930260:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575692.930267:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575692.930276:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575692.930283:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575692.930289:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575692.930294:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575692.930303:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.930312:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575692.930316:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575692.930321:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.930326:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575692.930334:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575692.930343:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575692.930352:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575692.930361:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575692.930366:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575692.930370:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575692.930374:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575692.930381:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575692.930391:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575692.930406:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516854784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575692.930425:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575692.930431:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575692.930442:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516854784/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575692.930472:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575692.930480:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16390) 00000100:00000001:0.0:1788575693.440932:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575693.440943:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575693.440947:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575693.440955:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575693.440963:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575693.440968:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575693.440982:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575693.440985:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575693.440988:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575695.488951:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575695.488956:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575695.488960:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575695.488967:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575695.488973:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575695.488977:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575695.490209:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575695.490213:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575695.490222:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575696.194382:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575696.194385:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575696.194388:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575696.512979:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575696.513004:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575696.513007:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575696.513014:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575696.513020:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575696.513023:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575696.513082:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575696.513084:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575696.513087:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575696.513112:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575696.513114:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575696.513116:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575696.513118:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575696.513120:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575696.513123:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575696.513125:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575696.513126:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575696.513128:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575696.768949:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311056622, at 4311056640 00000100:00000010:0.0:1788575696.768971:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf480. 00000020:00000001:0.0:1788575697.221603:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575697.221607:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575697.221610:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575697.536973:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575697.536983:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575697.536987:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575697.536994:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575697.537002:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575697.537009:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575697.537019:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575697.537023:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575697.537035:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575697.537076:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575697.537080:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575697.537081:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575697.537084:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575697.537086:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575697.537088:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575697.537941:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575697.537945:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575697.537949:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575698.048973:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575698.048986:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575698.048990:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575698.048996:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575698.049002:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575698.049010:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16385 00000400:00000200:0.0:1788575698.049022:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575698.049029:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575698.049033:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575698.049036:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049039:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575698.049042:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049044:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575698.049048:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575698.049051:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575698.049065:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575698.049069:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575698.049071:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575698.049073:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575698.049077:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049079:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575698.049081:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049083:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575698.049086:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575698.049089:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049091:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575698.049094:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575698.049098:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575698.049123:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.049126:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575698.049129:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575698.049138:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575698.049142:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575698.049147:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575698.049149:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.049152:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575698.049156:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575698.049161:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575698.049170:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575698.049179:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575698.049189:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516854912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575698.049200:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575698.049206:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575698.049212:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516854912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575698.049235:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575698.049239:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575698.049243:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575698.049244:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575698.049246:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575698.049249:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575698.049252:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16385 00000400:00000200:0.0:1788575698.049257:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575698.049263:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575698.049267:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575698.049270:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049272:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575698.049276:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049287:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575698.049293:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575698.049296:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575698.049299:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575698.049304:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575698.049309:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575698.049317:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575698.049331:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049337:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575698.049343:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049348:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575698.049354:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575698.049362:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049370:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575698.049376:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575698.049384:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575698.049390:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.049398:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575698.049406:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575698.049413:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.049418:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575698.049425:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575698.049430:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575698.049436:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575698.049441:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.049444:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575698.049447:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575698.049453:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575698.049461:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575698.049468:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575698.049481:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516855040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575698.049542:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575698.049545:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575698.049554:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516855040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575698.049573:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575698.049579:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575698.049593:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575698.049595:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575698.049597:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575698.049605:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575698.049614:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16385 00000400:00000200:0.0:1788575698.049624:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575698.049633:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575698.049635:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575698.049643:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049650:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575698.049657:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049663:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575698.049670:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575698.049674:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575698.049682:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575698.049688:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575698.049693:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575698.049700:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575698.049711:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049717:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575698.049725:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049731:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575698.049752:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575698.049759:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575698.049762:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575698.049764:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575698.049768:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575698.049771:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.049774:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575698.049778:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575698.049785:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.049793:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575698.049801:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575698.049854:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575698.049862:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575698.049867:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.049872:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575698.049875:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575698.049878:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575698.049890:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575698.049901:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575698.049913:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516855168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575698.049950:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575698.049955:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575698.049964:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516855168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575698.049993:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575698.050000:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575698.050009:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575698.050013:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575698.050017:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575698.050024:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575698.050038:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16385 00000400:00000200:0.0:1788575698.050063:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575698.050073:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575698.050078:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575698.050087:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.050095:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575698.050100:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.050107:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575698.050114:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575698.050117:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575698.050122:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575698.050126:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575698.050132:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575698.050138:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575698.050149:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.050154:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575698.050160:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.050166:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575698.050173:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575698.050180:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575698.050187:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575698.050195:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575698.050202:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575698.050209:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.050215:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575698.050222:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575698.050229:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.050236:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575698.050245:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575698.050253:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575698.050261:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575698.050271:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.050278:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575698.050283:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575698.050288:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575698.050297:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575698.050303:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575698.050314:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516855296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575698.050339:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575698.050345:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575698.050358:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516855296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575698.050390:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575698.050396:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575698.050404:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575698.050409:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575698.050413:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575698.050418:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575698.050425:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16385 00000400:00000200:0.0:1788575698.050442:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575698.050453:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575698.050460:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575698.050470:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.050481:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575698.050488:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.050494:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575698.050502:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575698.050506:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575698.050511:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575698.050515:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575698.050520:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575698.050533:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575698.050544:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.050550:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575698.050558:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575698.050562:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575698.050568:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575698.050576:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575698.050585:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575698.050590:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575698.050596:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575698.050603:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.050609:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575698.050613:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575698.050623:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.050630:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575698.050641:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575698.050647:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575698.050654:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575698.050657:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575698.050661:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575698.050665:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575698.050672:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575698.050684:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575698.050694:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575698.050705:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516855424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575698.050728:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575698.050731:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575698.050736:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516855424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575698.050746:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575698.050757:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16395) 00000100:00000001:0.0:1788575698.561014:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575698.561021:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575698.561024:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575698.561029:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575698.561034:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575698.561037:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575698.561041:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575698.561043:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575698.561046:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575700.609114:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575700.609118:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575700.609120:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575700.609123:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575700.609125:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575700.609127:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575700.609129:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575700.609131:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575700.609133:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575701.313081:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575701.313084:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575701.313087:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575701.632944:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575701.632949:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575701.632952:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575701.632965:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575701.632970:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575701.632973:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575701.632976:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575701.632978:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575701.632980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575701.888905:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311061741, at 4311061760 00000100:00000010:0.0:1788575701.888920:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf660. 00000020:00000001:0.0:1788575702.337347:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575702.337349:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575702.337351:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575702.656926:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575702.656944:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575702.656947:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575702.656950:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575702.656953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575702.656955:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575702.656972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575702.656976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575702.656979:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.169998:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575703.170004:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575703.170006:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575703.170014:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516855552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.170025:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575703.170027:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575703.170031:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575703.170035:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575703.170038:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575703.170041:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516855552:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575703.170046:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575703.170066:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575703.170069:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575703.170072:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575703.170079:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575703.170081:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575703.170088:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575703.170093:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575703.170103:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575703.170112:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516855552, portal 25 00000100:00000001:0.0:1788575703.170311:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575703.170313:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575703.170321:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516855552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.170329:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575703.170349:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575703.170352:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575703.170356:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516855552, offset 0 00000400:00000200:0.0:1788575703.170363:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575703.170373:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575703.170379:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575703.170386:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575703.170391:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575703.170394:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575703.170397:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575703.170401:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548336:548336:256:4294967295] 192.168.202.128@tcp LPNI seq info [548259:548259:8:4294967295] 00000400:00000200:0.0:1788575703.170405:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575703.170414:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575703.170421:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575703.170427:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575703.170435:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575703.170438:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788575703.170458:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575703.170461:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575703.170466:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575703.170471:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575703.170477:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575703.170479:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575703.170485:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575703.170490:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516855552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.170500:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575703.170504:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516855552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.170511:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.170513:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575703.170527:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575703.170532:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575703.170535:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311066862 00000100:00000010:0.0:1788575703.170539:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575703.170546:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575703.170553:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311066862 00000100:00000010:0.0:1788575703.170556:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b504e0. 00000100:00000001:0.0:1788575703.170559:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575703.170563:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575703.170566:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.170568:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575703.170573:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575703/real 1788575703] req@ffff8fc4901a4a80 x1875440516855552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575703.170582:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575703.170584:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575703.170586:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575703.170588:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575703.170590:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575703.170593:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575703.170595:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516855552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575703.170603:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516855552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.170609:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575703.170611:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.170613:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575703.170615:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575703.170619:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516855552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.170631:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575703.170636:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516855552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.170648:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575703.170651:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.170653:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575703.170654:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.170656:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575703.170658:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575703.170661:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575703.170665:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575703.170667:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575703.170671:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516855552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575703.170679:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516855552:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575703.170683:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575703.170686:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516855680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.170692:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575703.170693:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575703.170696:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575703.170709:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575703.170711:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575703.170714:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516855680:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575703.170717:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575703.170720:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575703.170722:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575703.170723:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575703.170727:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575703.170728:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575703.170732:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575703.170742:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575703.170745:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575703.170748:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516855680, portal 10 00000100:00000001:0.0:1788575703.170750:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575703.170751:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575703.170758:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516855680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.170764:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575703.170766:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575703.170768:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575703.170770:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516855680, offset 0 00000400:00000200:0.0:1788575703.170774:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575703.170778:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575703.170782:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575703.170785:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575703.170787:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575703.170789:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575703.170791:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575703.170795:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548337:548337:256:4294967295] 192.168.202.128@tcp LPNI seq info [548260:548260:8:4294967295] 00000400:00000200:0.0:1788575703.170799:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575703.170805:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575703.170846:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575703.170850:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575703.170854:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575703.170856:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788575703.170867:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575703.170869:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575703.170872:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575703.170875:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575703.170878:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575703.170880:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575703.170889:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575703.170893:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516855680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.170900:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575703.170903:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516855680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.170910:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.170911:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575703.170915:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575703.170918:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575703.170922:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575703.170924:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311063060 00000100:00000001:0.0:1788575703.170931:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311066862 00000100:00000001:0.0:1788575703.170934:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311063060 00000100:00000001:0.0:1788575703.170936:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311066862 00000100:00000001:0.0:1788575703.170939:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575703.170941:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.170943:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575703.170947:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575703/real 1788575703] req@ffff8fc4901a7100 x1875440516855680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575703.170954:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575703.170956:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575703.170958:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575703.170960:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575703.170962:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575703.170964:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575703.170967:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516855680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575703.170975:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516855680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.170992:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575703.170994:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.170995:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575703.170997:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575703.171001:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516855680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171007:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575703.171011:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516855680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171016:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575703.171019:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171020:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575703.171021:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171023:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575703.171026:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575703.171028:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575703.171031:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575703.171033:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575703.171037:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516855680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575703.171045:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516855680:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575703.171049:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575703.171071:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516855808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171077:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575703.171078:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575703.171087:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575703.171089:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575703.171091:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575703.171094:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516855808:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575703.171097:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575703.171099:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575703.171100:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575703.171102:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575703.171105:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575703.171107:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575703.171110:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575703.171113:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575703.171116:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575703.171119:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516855808, portal 10 00000100:00000001:0.0:1788575703.171121:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575703.171122:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575703.171127:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516855808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171133:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575703.171134:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575703.171136:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575703.171138:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516855808, offset 0 00000400:00000200:0.0:1788575703.171141:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575703.171145:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575703.171148:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575703.171150:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575703.171153:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575703.171155:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575703.171156:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575703.171159:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548338:548338:256:4294967295] 192.168.202.128@tcp LPNI seq info [548261:548261:8:4294967295] 00000400:00000200:0.0:1788575703.171162:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575703.171168:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575703.171177:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575703.171181:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575703.171193:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575703.171196:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788575703.171211:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575703.171213:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575703.171216:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575703.171219:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575703.171222:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575703.171224:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575703.171227:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575703.171230:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516855808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171237:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575703.171239:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516855808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171245:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.171246:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575703.171250:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575703.171253:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575703.171256:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575703.171258:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311063067 00000100:00000001:0.0:1788575703.171260:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311066862 00000100:00000001:0.0:1788575703.171262:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311063067 00000100:00000001:0.0:1788575703.171264:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311066862 00000100:00000001:0.0:1788575703.171266:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575703.171268:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.171270:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575703.171273:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575703/real 1788575703] req@ffff8fc4901a5180 x1875440516855808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575703.171286:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575703.171287:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575703.171289:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575703.171291:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575703.171292:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575703.171295:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575703.171298:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516855808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575703.171305:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516855808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171311:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575703.171312:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171314:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575703.171315:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575703.171318:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516855808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171323:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575703.171326:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516855808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171332:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575703.171334:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171336:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575703.171337:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171339:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575703.171341:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575703.171344:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575703.171346:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575703.171348:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575703.171356:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516855808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575703.171361:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516855808:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575703.171363:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575703.171365:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516855936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171369:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575703.171370:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575703.171371:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575703.171372:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575703.171374:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575703.171375:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516855936:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575703.171377:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575703.171379:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575703.171380:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575703.171381:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575703.171383:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575703.171384:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575703.171386:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575703.171388:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575703.171390:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575703.171392:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516855936, portal 4 00000100:00000001:0.0:1788575703.171394:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575703.171395:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575703.171398:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516855936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171402:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575703.171403:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575703.171405:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575703.171406:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516855936, offset 0 00000400:00000200:0.0:1788575703.171411:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575703.171414:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575703.171416:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575703.171418:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575703.171419:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575703.171421:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575703.171422:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575703.171424:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548339:548339:256:4294967295] 192.168.202.128@tcp LPNI seq info [548262:548262:8:4294967295] 00000400:00000200:0.0:1788575703.171427:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575703.171431:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575703.171434:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575703.171435:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575703.171438:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575703.171439:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788575703.171446:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575703.171447:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575703.171449:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575703.171451:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575703.171453:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575703.171454:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575703.171455:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575703.171457:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516855936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171462:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575703.171464:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516855936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171468:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.171468:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575703.171470:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575703.171472:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575703.171474:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575703.171497:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311063207 00000100:00000001:0.0:1788575703.171499:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311066862 00000100:00000001:0.0:1788575703.171501:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311063207 00000100:00000001:0.0:1788575703.171502:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311066862 00000100:00000001:0.0:1788575703.171504:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575703.171505:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.171506:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575703.171509:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575703/real 1788575703] req@ffff8fc4901a5c00 x1875440516855936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575703.171514:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575703.171515:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575703.171516:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575703.171518:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575703.171519:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575703.171520:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575703.171522:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516855936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575703.171527:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516855936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171531:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575703.171531:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171532:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575703.171533:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575703.171535:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516855936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171539:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575703.171541:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516855936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171545:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575703.171549:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171551:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575703.171551:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171553:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575703.171554:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575703.171556:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575703.171558:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575703.171559:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575703.171561:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516855936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575703.171566:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516855936:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575703.171568:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575703.171570:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516856064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171574:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575703.171575:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575703.171576:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575703.171577:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575703.171578:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575703.171580:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516856064:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575703.171581:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575703.171582:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575703.171583:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575703.171585:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575703.171588:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575703.171589:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575703.171591:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575703.171592:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575703.171594:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575703.171596:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516856064, portal 4 00000100:00000001:0.0:1788575703.171600:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575703.171601:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575703.171605:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516856064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171609:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575703.171611:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575703.171612:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575703.171614:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516856064, offset 0 00000400:00000200:0.0:1788575703.171616:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575703.171618:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575703.171620:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575703.171622:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575703.171624:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575703.171625:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575703.171627:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575703.171629:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548340:548340:256:4294967295] 192.168.202.128@tcp LPNI seq info [548263:548263:8:4294967295] 00000400:00000200:0.0:1788575703.171631:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575703.171635:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575703.171638:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575703.171640:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575703.171642:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575703.171644:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788575703.171657:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575703.171659:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575703.171662:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575703.171665:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575703.171668:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575703.171669:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575703.171672:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575703.171675:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516856064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171693:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575703.171696:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516856064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575728 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171703:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.171704:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575703.171707:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575703.171709:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575703.171712:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575703.171713:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311063209 00000100:00000001:0.0:1788575703.171714:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311066862 00000100:00000001:0.0:1788575703.171716:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311063209 00000100:00000001:0.0:1788575703.171717:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311066862 00000100:00000001:0.0:1788575703.171719:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575703.171720:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.171721:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575703.171724:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575703/real 1788575703] req@ffff8fc4901a7800 x1875440516856064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575703.171728:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575703.171729:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575703.171730:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575703.171731:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575703.171732:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575703.171734:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575703.171736:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516856064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575703.171741:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516856064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171745:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575703.171749:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171750:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575703.171751:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575703.171754:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516856064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171758:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575703.171761:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516856064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171765:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575703.171766:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171767:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575703.171768:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171769:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575703.171770:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575703.171771:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575703.171773:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575703.171774:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575703.171776:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516856064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575703.171780:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516856064:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575703.171782:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171783:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575703.171786:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516855552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171791:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575703.171792:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575703.171795:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575703.171797:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575703.171799:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575703.171803:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575703.171805:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575703.171807:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575703.171837:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575703.171839:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575703.171841:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575703.171846:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575703.171847:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171849:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575703.171853:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516855680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171859:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575703.171861:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575703.171862:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575703.171865:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575703.171867:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575703.171868:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575703.171870:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575703.171871:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575703.171873:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575703.171874:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575703.171876:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575703.171878:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575703.171880:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171881:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575703.171884:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516855808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171890:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575703.171891:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575703.171893:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575703.171895:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575703.171896:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575703.171897:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575703.171899:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575703.171900:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575703.171903:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575703.171909:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575703.171910:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575703.171915:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575703.171916:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171917:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575703.171919:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516855936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171924:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575703.171924:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575703.171926:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575703.171927:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575703.171928:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575703.171929:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575703.171930:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575703.171931:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575703.171932:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575703.171933:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575703.171934:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575703.171935:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575703.171936:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171937:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575703.171938:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516856064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575728 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575703.171942:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575703.171942:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575703.171943:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575703.171944:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575703.171945:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575703.171945:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575703.171946:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575703.171947:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575703.171948:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575703.171948:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575703.171949:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575703.171950:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575703.171951:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171955:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.171961:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575703.171963:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.171965:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575703.171967:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575703.681213:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575703.681216:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575703.681217:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575703.681220:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575703.681222:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575703.681224:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.681372:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575703.681374:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575703.681377:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575704.704946:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575704.704953:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575704.704956:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575704.704964:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575704.704971:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575704.704976:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575704.704981:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575704.704984:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575704.704995:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575704.705078:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575704.705081:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575704.705083:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575704.705086:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575704.705089:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575704.705091:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575704.705093:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575704.705094:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575704.705096:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575705.728933:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575705.728942:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575705.728945:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575705.728950:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575705.728955:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575705.728959:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575705.728980:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575705.728985:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575705.729002:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575706.433002:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575706.433006:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575706.433009:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575706.753068:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575706.753072:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575706.753073:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575706.753076:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575706.753080:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575706.753082:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575706.753326:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575706.753330:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575706.753334:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575707.009076:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311066862, at 4311066880 00000100:00000010:0.0:1788575707.009082:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575707.457001:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575707.457005:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575707.457009:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575707.776909:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575707.776919:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575707.776922:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575707.776927:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575707.776932:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575707.776936:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575707.776938:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575707.776941:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575707.776944:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575707.776975:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575707.776981:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575707.776986:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575707.776989:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575707.776994:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575707.776997:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575707.777012:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575707.777014:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575707.777031:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.289156:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575708.289164:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575708.289168:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575708.289178:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516856192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.289193:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575708.289196:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575708.289201:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575708.289204:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575708.289208:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575708.289214:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516856192:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575708.289223:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575708.289228:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575708.289233:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575708.289238:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575708.289245:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575708.289249:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575708.289258:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575708.289264:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575708.289271:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575708.289279:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516856192, portal 25 00000100:00000001:0.0:1788575708.289283:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575708.289285:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575708.289292:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516856192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.289301:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575708.289308:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575708.289312:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575708.289316:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516856192, offset 0 00000400:00000200:0.0:1788575708.289324:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575708.289333:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575708.289341:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575708.289358:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575708.289363:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575708.289370:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575708.289377:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575708.289385:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548341:548341:256:4294967295] 192.168.202.128@tcp LPNI seq info [548264:548264:8:4294967295] 00000400:00000200:0.0:1788575708.289397:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575708.289414:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575708.289423:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575708.289429:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575708.289438:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575708.289444:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575708.289471:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575708.289476:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575708.289484:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575708.289493:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575708.289499:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575708.289502:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575708.289509:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575708.289515:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516856192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.289527:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575708.289533:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516856192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.289547:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.289551:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575708.289558:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575708.289567:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575708.289572:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311071981 00000100:00000010:0.0:1788575708.289579:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575708.289586:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575708.289596:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311071981 00000100:00000010:0.0:1788575708.289601:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50960. 00000100:00000001:0.0:1788575708.289605:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575708.289608:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575708.289611:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.289615:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575708.289623:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575708/real 1788575708] req@ffff8fc4901a5880 x1875440516856192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575708.289651:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575708.289655:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575708.289660:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575708.289666:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575708.289671:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575708.289677:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575708.289683:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516856192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575708.289697:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516856192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.289708:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575708.289711:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.289715:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575708.289718:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575708.289722:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516856192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.289733:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575708.289741:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516856192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.289768:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575708.289774:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.289779:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575708.289796:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.289801:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575708.289829:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575708.289832:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575708.289837:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575708.289839:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575708.289844:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516856192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575708.289851:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516856192:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575708.289857:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.289859:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575708.289862:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516856192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.289868:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575708.289870:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575708.289872:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575708.289875:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575708.289877:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575708.289878:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575708.289880:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575708.289882:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575708.289886:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575708.289889:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575708.289893:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575708.289901:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575708.289905:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.289909:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.289917:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575708.289922:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.289926:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575708.289930:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575708.289933:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575708.289940:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516856320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.289976:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575708.289979:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575708.289983:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575708.289986:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575708.289992:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575708.289999:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516856320:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575708.290003:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575708.290006:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575708.290009:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575708.290011:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575708.290014:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575708.290017:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575708.290024:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575708.290029:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575708.290033:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575708.290037:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516856320, portal 10 00000100:00000001:0.0:1788575708.290041:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575708.290043:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575708.290067:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516856320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290078:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575708.290085:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575708.290091:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575708.290096:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516856320, offset 0 00000400:00000200:0.0:1788575708.290107:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575708.290114:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575708.290120:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575708.290126:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575708.290128:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575708.290133:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575708.290136:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575708.290146:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548342:548342:256:4294967295] 192.168.202.128@tcp LPNI seq info [548265:548265:8:4294967295] 00000400:00000200:0.0:1788575708.290151:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575708.290162:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575708.290168:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575708.290173:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575708.290178:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575708.290181:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575708.290190:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575708.290191:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575708.290194:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575708.290197:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575708.290199:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575708.290200:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575708.290202:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575708.290205:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516856320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290212:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575708.290214:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516856320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290220:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.290221:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575708.290225:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575708.290228:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575708.290231:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575708.290233:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311068179 00000100:00000001:0.0:1788575708.290235:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311071981 00000100:00000001:0.0:1788575708.290237:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311068179 00000100:00000001:0.0:1788575708.290239:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311071981 00000100:00000001:0.0:1788575708.290241:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575708.290248:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.290250:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575708.290254:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575708/real 1788575708] req@ffff8fc4901a4700 x1875440516856320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575708.290260:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575708.290261:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575708.290262:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575708.290264:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575708.290265:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575708.290266:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575708.290268:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516856320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575708.290273:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516856320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290277:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575708.290277:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.290278:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575708.290279:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575708.290281:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516856320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290285:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575708.290287:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516856320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290291:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575708.290293:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.290293:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575708.290294:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.290295:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575708.290296:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575708.290298:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575708.290303:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575708.290304:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575708.290307:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516856320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575708.290311:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516856320:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575708.290314:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575708.290315:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516856448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290319:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575708.290320:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575708.290321:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575708.290322:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575708.290323:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575708.290325:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516856448:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575708.290326:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575708.290328:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575708.290329:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575708.290330:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575708.290332:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575708.290333:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575708.290335:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575708.290336:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575708.290338:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575708.290349:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516856448, portal 10 00000100:00000001:0.0:1788575708.290351:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575708.290352:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575708.290362:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516856448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290367:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575708.290376:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575708.290378:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575708.290381:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516856448, offset 0 00000400:00000200:0.0:1788575708.290385:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575708.290389:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575708.290393:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575708.290397:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575708.290399:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575708.290401:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575708.290403:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575708.290406:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548343:548343:256:4294967295] 192.168.202.128@tcp LPNI seq info [548266:548266:8:4294967295] 00000400:00000200:0.0:1788575708.290409:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575708.290415:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575708.290420:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575708.290423:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575708.290427:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575708.290430:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575708.290439:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575708.290440:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575708.290443:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575708.290447:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575708.290450:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575708.290452:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575708.290454:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575708.290458:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516856448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290465:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575708.290468:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516856448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290475:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.290481:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575708.290486:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575708.290489:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575708.290493:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575708.290494:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311068186 00000100:00000001:0.0:1788575708.290497:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311071981 00000100:00000001:0.0:1788575708.290500:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311068186 00000100:00000001:0.0:1788575708.290502:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311071981 00000100:00000001:0.0:1788575708.290504:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575708.290506:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.290508:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575708.290512:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575708/real 1788575708] req@ffff8fc4901a7480 x1875440516856448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575708.290519:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575708.290521:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575708.290522:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575708.290525:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575708.290527:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575708.290529:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575708.290533:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516856448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575708.290541:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516856448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290548:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575708.290549:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.290551:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575708.290552:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575708.290555:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516856448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290561:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575708.290572:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516856448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290579:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575708.290582:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.290583:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575708.290585:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.290586:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575708.290589:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575708.290592:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575708.290595:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575708.290597:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575708.290600:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516856448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575708.290608:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516856448:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575708.290612:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575708.290615:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516856576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290622:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575708.290623:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575708.290626:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575708.290627:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575708.290629:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575708.290632:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516856576:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575708.290635:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575708.290637:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575708.290638:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575708.290640:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575708.290643:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575708.290645:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575708.290654:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575708.290656:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575708.290660:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575708.290663:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516856576, portal 4 00000100:00000001:0.0:1788575708.290665:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575708.290666:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575708.290672:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516856576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290678:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575708.290680:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575708.290682:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575708.290685:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516856576, offset 0 00000400:00000200:0.0:1788575708.290688:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575708.290692:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575708.290696:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575708.290698:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575708.290701:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575708.290704:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575708.290706:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575708.290709:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548344:548344:256:4294967295] 192.168.202.128@tcp LPNI seq info [548267:548267:8:4294967295] 00000400:00000200:0.0:1788575708.290713:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575708.290719:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575708.290724:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575708.290728:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575708.290732:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575708.290735:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575708.290745:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575708.290747:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575708.290749:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575708.290753:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575708.290756:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575708.290762:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575708.290766:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575708.290769:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516856576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290776:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575708.290779:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516856576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290785:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.290787:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575708.290790:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575708.290793:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575708.290796:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575708.290798:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311068326 00000100:00000001:0.0:1788575708.290801:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311071981 00000100:00000001:0.0:1788575708.290803:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311068326 00000100:00000001:0.0:1788575708.290805:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311071981 00000100:00000001:0.0:1788575708.290826:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575708.290828:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.290830:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575708.290834:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575708/real 1788575708] req@ffff8fc4901a6300 x1875440516856576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575708.290843:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575708.290844:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575708.290846:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575708.290849:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575708.290850:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575708.290852:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575708.290855:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516856576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575708.290863:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516856576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290874:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575708.290876:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.290878:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575708.290879:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575708.290883:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516856576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290889:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575708.290893:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516856576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290900:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575708.290903:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.290904:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575708.290906:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.290908:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575708.290910:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575708.290912:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575708.290915:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575708.290918:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575708.290922:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516856576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575708.290929:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516856576:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575708.290933:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.290935:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575708.290938:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516856320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290945:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575708.290946:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575708.290952:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575708.290955:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575708.290956:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575708.290958:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575708.290960:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575708.290961:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575708.290963:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575708.290965:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575708.290966:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575708.290970:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575708.290972:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.290973:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575708.290977:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516856448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.290983:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575708.290984:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575708.290985:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575708.290987:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575708.290989:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575708.290990:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575708.290992:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575708.290993:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575708.290995:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575708.290996:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575708.290997:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575708.291000:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575708.291002:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.291003:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575708.291006:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516856576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.291012:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575708.291014:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575708.291015:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575708.291017:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575708.291018:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575708.291020:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575708.291026:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575708.291028:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575708.291030:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575708.291031:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575708.291033:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575708.291037:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575708.291038:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.291040:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.291043:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575708.291045:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.291046:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575708.291048:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575708.291062:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575708.291067:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516856704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.291074:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575708.291075:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575708.291077:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575708.291079:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575708.291081:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575708.291084:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516856704:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575708.291087:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575708.291089:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575708.291091:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575708.291092:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575708.291095:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575708.291097:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575708.291100:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575708.291103:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575708.291106:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575708.291109:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516856704, portal 4 00000100:00000001:0.0:1788575708.291111:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575708.291113:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575708.291118:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516856704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.291133:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575708.291136:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575708.291138:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575708.291141:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516856704, offset 0 00000400:00000200:0.0:1788575708.291145:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575708.291149:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575708.291152:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575708.291155:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575708.291158:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575708.291160:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575708.291162:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575708.291166:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548345:548345:256:4294967295] 192.168.202.128@tcp LPNI seq info [548268:548268:8:4294967295] 00000400:00000200:0.0:1788575708.291170:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575708.291176:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575708.291181:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575708.291184:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575708.291188:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575708.291190:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575708.291201:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575708.291203:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575708.291206:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575708.291209:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575708.291212:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575708.291214:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575708.291217:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575708.291220:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516856704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.291228:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575708.291232:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516856704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575733 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.291249:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.291251:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575708.291255:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575708.291258:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575708.291262:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575708.291264:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311068329 00000100:00000001:0.0:1788575708.291266:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311071981 00000100:00000001:0.0:1788575708.291268:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311068329 00000100:00000001:0.0:1788575708.291270:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311071981 00000100:00000001:0.0:1788575708.291272:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575708.291274:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.291276:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575708.291279:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575708/real 1788575708] req@ffff8fc4901a4e00 x1875440516856704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575708.291287:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575708.291288:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575708.291289:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575708.291291:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575708.291293:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575708.291295:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575708.291298:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516856704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575708.291306:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516856704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.291312:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575708.291314:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.291316:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575708.291317:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575708.291321:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516856704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.291332:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575708.291336:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516856704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.291342:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575708.291345:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.291346:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575708.291348:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.291350:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575708.291352:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575708.291354:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575708.291358:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575708.291360:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575708.291364:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516856704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575708.291372:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516856704:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575708.291376:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.291378:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575708.291380:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516856704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575733 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575708.291386:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575708.291387:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575708.291389:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575708.291392:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575708.291393:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575708.291394:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575708.291396:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575708.291398:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575708.291400:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575708.291406:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575708.291407:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575708.291412:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575708.291413:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.291415:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.291418:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575708.291420:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.291421:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575708.291423:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.800932:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575708.800939:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575708.800948:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575708.800953:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575708.800957:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575708.800961:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.800964:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575708.800966:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.800969:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575708.801926:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575708.801929:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575708.801930:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575708.801933:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575708.801935:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575708.801937:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.801939:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575708.801941:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575708.801943:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575709.824958:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575709.824963:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575709.824966:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575709.824971:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575709.824975:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575709.824979:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575709.824982:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575709.824986:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575709.824989:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575709.825010:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575709.825014:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575709.825016:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575709.825035:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575709.825038:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575709.825041:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575709.825043:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575709.825045:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575709.825057:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575710.848912:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575710.848919:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575710.848921:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575710.848926:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575710.848930:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575710.848932:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575710.848935:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575710.848937:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575710.848940:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575711.552995:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575711.552998:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575711.553006:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575711.872936:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575711.872945:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575711.872949:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575711.872957:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575711.872963:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575711.872969:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575711.872976:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575711.872979:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575711.872981:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575711.873011:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575711.873015:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575711.873017:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575711.873020:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575711.873031:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575711.873034:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575711.873036:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575711.873038:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575711.873040:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575712.128903:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311071981, at 4311072000 00000100:00000010:0.0:1788575712.128917:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575712.577019:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575712.577022:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575712.577024:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575712.896915:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575712.896923:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575712.896928:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575712.896937:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575712.896945:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575712.896953:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575712.896981:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575712.896985:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575712.896988:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575713.920931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575713.920936:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575713.920939:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575713.920944:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575713.920949:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575713.920954:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575713.923378:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575713.923382:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575713.923386:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575714.945045:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575714.945072:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575714.945075:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575714.945081:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575714.945086:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575714.945090:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575714.945094:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575714.945097:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575714.945102:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575715.968926:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575715.968931:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575715.968934:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575715.968939:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575715.968945:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575715.968949:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575715.968958:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575715.968961:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575715.968978:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575716.674093:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575716.674098:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575716.674101:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575716.992919:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575716.992924:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575716.992926:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575716.992929:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575716.992946:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575716.992949:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575716.993045:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575716.993048:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575716.993050:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575717.248911:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311077102, at 4311077120 00000100:00000010:0.0:1788575717.248925:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf7e0. 00000020:00000001:0.0:1788575717.697143:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575717.697147:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575717.697151:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575718.016932:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575718.016938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575718.016957:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575718.016962:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575718.016967:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575718.016970:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575718.016988:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575718.016993:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575718.016998:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575718.528987:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575718.529000:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575718.529003:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575718.529007:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575718.529012:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575718.529020:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16405 00000400:00000200:0.0:1788575718.529031:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575718.529038:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575718.529068:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575718.529073:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529077:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575718.529082:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529085:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575718.529090:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575718.529093:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575718.529096:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575718.529099:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575718.529102:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575718.529105:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575718.529110:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529112:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575718.529116:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529118:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575718.529122:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575718.529131:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529137:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575718.529141:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575718.529146:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575718.529150:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.529152:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575718.529155:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575718.529163:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575718.529167:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575718.529171:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575718.529173:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.529176:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575718.529179:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575718.529184:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575718.529192:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575718.529200:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575718.529217:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516857472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575718.529228:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575718.529233:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575718.529238:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516857472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575718.529263:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575718.529266:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575718.529270:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575718.529271:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575718.529273:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575718.529275:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575718.529278:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16405 00000400:00000200:0.0:1788575718.529283:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575718.529287:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575718.529290:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575718.529294:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529296:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575718.529300:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529303:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575718.529308:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575718.529310:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575718.529315:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575718.529318:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575718.529322:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575718.529325:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575718.529333:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529341:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575718.529354:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529368:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575718.529377:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575718.529383:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529386:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575718.529388:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575718.529395:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575718.529400:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.529404:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575718.529411:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575718.529417:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.529421:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575718.529427:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575718.529432:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575718.529437:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575718.529438:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.529441:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575718.529445:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575718.529449:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575718.529456:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575718.529462:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575718.529471:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516857600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575718.529487:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575718.529490:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575718.529497:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516857600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575718.529520:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575718.529527:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575718.529542:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575718.529545:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575718.529548:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575718.529559:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575718.529566:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16405 00000400:00000200:0.0:1788575718.529575:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575718.529586:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575718.529591:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575718.529597:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529600:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575718.529607:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529615:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575718.529618:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575718.529619:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575718.529622:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575718.529626:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575718.529631:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575718.529633:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575718.529640:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529644:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575718.529648:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529653:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575718.529656:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575718.529663:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529666:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575718.529670:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575718.529676:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575718.529683:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.529686:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575718.529690:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575718.529696:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.529701:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575718.529707:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575718.529715:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575718.529720:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575718.529723:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.529727:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575718.529731:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575718.529735:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575718.529743:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575718.529750:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575718.529761:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516857728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575718.529785:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575718.529791:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575718.529804:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516857728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575718.529866:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575718.529875:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575718.529880:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575718.529883:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575718.529888:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575718.529892:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575718.529899:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16405 00000400:00000200:0.0:1788575718.529909:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575718.529933:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575718.529938:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575718.529946:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529954:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575718.529964:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.529970:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575718.529978:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575718.529980:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575718.529990:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575718.529993:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575718.530001:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575718.530005:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575718.530016:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.530023:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575718.530025:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.530028:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575718.530034:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575718.530037:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575718.530045:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575718.530048:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575718.530053:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575718.530062:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.530067:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575718.530077:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575718.530087:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.530093:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575718.530102:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575718.530113:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575718.530121:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575718.530126:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.530131:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575718.530137:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575718.530142:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575718.530152:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575718.530160:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575718.530176:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516857856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575718.530196:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575718.530201:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575718.530215:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516857856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575718.530232:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575718.530237:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575718.530243:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575718.530248:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575718.530254:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575718.530267:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575718.530277:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16405 00000400:00000200:0.0:1788575718.530291:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575718.530306:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575718.530314:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575718.530328:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.530337:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575718.530347:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.530354:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575718.530361:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575718.530364:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575718.530368:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575718.530372:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575718.530379:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575718.530385:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575718.530397:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.530402:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575718.530407:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575718.530414:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575718.530417:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575718.530423:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575718.530427:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575718.530439:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575718.530444:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575718.530453:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.530464:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575718.530471:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575718.530484:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.530488:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575718.530497:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575718.530506:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575718.530512:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575718.530514:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575718.530519:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575718.530523:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575718.530528:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575718.530539:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575718.530547:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575718.530559:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516857984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575718.530579:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575718.530585:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575718.530597:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516857984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575718.530639:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575718.530652:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16416) 00000100:00000001:0.0:1788575719.040933:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575719.040947:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575719.040953:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575719.040971:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575719.040984:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575719.040993:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575719.041016:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575719.041020:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575719.041024:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575719.041155:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575719.041179:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575719.041181:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575719.041183:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575719.041186:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575719.041188:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575719.041190:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575719.041191:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575719.041194:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575720.064991:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575720.065003:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575720.065009:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575720.065019:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575720.065030:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575720.065036:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575720.065053:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575720.065056:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575720.065061:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575721.088961:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575721.088973:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575721.088978:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575721.088989:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575721.088998:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575721.089008:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575721.089019:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575721.089024:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575721.089030:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575721.089074:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575721.089080:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575721.089083:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575721.089086:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575721.089089:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575721.089092:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575721.089094:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575721.089096:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575721.089099:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575721.793082:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575721.793090:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575721.793094:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575722.112941:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575722.112952:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575722.112961:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575722.112971:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575722.112979:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575722.112985:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575722.112993:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575722.113001:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575722.113006:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575722.113030:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575722.113035:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575722.113040:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575722.113044:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575722.113047:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575722.113050:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575722.113054:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575722.113056:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575722.113059:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575722.368910:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311082222, at 4311082240 00000100:00000010:0.0:1788575722.368929:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfc00. 00000020:00000001:0.0:1788575722.816968:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575722.816972:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575722.816974:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575723.136959:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575723.136968:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575723.136973:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575723.136978:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575723.136989:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575723.136996:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.137015:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575723.137018:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.137022:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575723.649135:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788575723.649148:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788575723.649154:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788575723.649367:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575723.649404:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575723.649410:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575723.649414:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.649491:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575723.649498:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575723.649503:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575723.649514:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516858112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.649529:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575723.649532:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575723.649538:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575723.649543:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575723.649545:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575723.649551:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516858112:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575723.649557:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575723.649561:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575723.649563:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575723.649566:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575723.649572:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575723.649574:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575723.649581:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575723.649586:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575723.649593:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575723.649601:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516858112, portal 25 00000100:00000001:0.0:1788575723.649606:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575723.649609:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575723.649615:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516858112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.649624:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575723.649628:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575723.649633:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575723.649637:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516858112, offset 0 00000400:00000200:0.0:1788575723.649654:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575723.649692:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575723.649710:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575723.649719:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575723.649727:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575723.649735:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575723.649739:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575723.649744:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548356:548356:256:4294967295] 192.168.202.128@tcp LPNI seq info [548279:548279:8:4294967295] 00000400:00000200:0.0:1788575723.649755:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575723.649774:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575723.649792:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575723.649803:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575723.649827:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575723.649829:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575723.649840:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575723.649843:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575723.649846:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575723.649853:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575723.649858:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575723.649862:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575723.649867:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575723.649873:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516858112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.649886:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575723.649892:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516858112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.649910:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.649912:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575723.649920:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575723.649928:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575723.649933:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311087342 00000100:00000010:0.0:1788575723.649941:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575723.649956:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575723.649964:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311087342 00000100:00000010:0.0:1788575723.649970:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50900. 00000100:00000001:0.0:1788575723.649974:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575723.649980:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575723.649983:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.649988:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575723.649995:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575723/real 1788575723] req@ffff8fc4901a4e00 x1875440516858112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575723.650018:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575723.650022:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575723.650027:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575723.650033:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575723.650036:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575723.650043:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575723.650049:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516858112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575723.650061:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516858112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650070:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575723.650073:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.650077:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575723.650080:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575723.650086:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516858112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650104:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575723.650112:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516858112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650127:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575723.650133:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.650143:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575723.650147:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.650151:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575723.650156:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575723.650161:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575723.650169:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575723.650172:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575723.650179:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516858112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575723.650192:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516858112:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575723.650201:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.650204:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575723.650209:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516858112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650220:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575723.650224:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575723.650227:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575723.650232:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575723.650235:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575723.650244:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575723.650249:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575723.650252:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575723.650257:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575723.650260:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575723.650262:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575723.650271:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575723.650273:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.650275:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.650284:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575723.650287:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.650289:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575723.650292:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575723.650294:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575723.650305:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516858240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650322:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575723.650324:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575723.650326:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575723.650330:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575723.650332:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575723.650339:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516858240:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575723.650342:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575723.650345:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575723.650347:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575723.650348:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575723.650351:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575723.650353:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575723.650357:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575723.650360:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575723.650363:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575723.650366:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516858240, portal 10 00000100:00000001:0.0:1788575723.650368:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575723.650369:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575723.650377:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516858240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650383:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575723.650385:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575723.650387:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575723.650389:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516858240, offset 0 00000400:00000200:0.0:1788575723.650391:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575723.650394:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575723.650396:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575723.650398:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575723.650400:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575723.650402:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575723.650409:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575723.650411:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548357:548357:256:4294967295] 192.168.202.128@tcp LPNI seq info [548280:548280:8:4294967295] 00000400:00000200:0.0:1788575723.650414:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575723.650418:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575723.650421:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575723.650423:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575723.650426:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575723.650427:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575723.650434:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575723.650435:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575723.650437:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575723.650439:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575723.650442:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575723.650443:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575723.650445:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575723.650447:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516858240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650452:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575723.650454:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516858240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650458:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.650459:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575723.650461:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575723.650463:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575723.650465:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575723.650467:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311083539 00000100:00000001:0.0:1788575723.650468:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311087342 00000100:00000001:0.0:1788575723.650470:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311083539 00000100:00000001:0.0:1788575723.650472:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311087342 00000100:00000001:0.0:1788575723.650478:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575723.650480:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.650481:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575723.650485:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575723/real 1788575723] req@ffff8fc4901a6300 x1875440516858240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575723.650493:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575723.650494:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575723.650496:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575723.650498:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575723.650500:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575723.650502:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575723.650505:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516858240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575723.650513:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516858240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650518:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575723.650519:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.650521:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575723.650522:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575723.650524:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516858240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650528:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575723.650530:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516858240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650534:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575723.650535:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.650536:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575723.650537:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.650538:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575723.650539:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575723.650544:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575723.650547:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575723.650548:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575723.650550:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516858240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575723.650555:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516858240:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575723.650557:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575723.650559:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516858368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650562:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575723.650563:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575723.650564:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575723.650566:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575723.650567:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575723.650569:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516858368:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575723.650570:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575723.650572:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575723.650572:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575723.650573:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575723.650575:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575723.650576:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575723.650579:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575723.650580:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575723.650582:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575723.650584:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516858368, portal 10 00000100:00000001:0.0:1788575723.650585:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575723.650586:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575723.650590:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516858368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650599:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575723.650601:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575723.650602:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575723.650604:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516858368, offset 0 00000400:00000200:0.0:1788575723.650607:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575723.650611:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575723.650614:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575723.650617:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575723.650619:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575723.650621:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575723.650623:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575723.650638:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548358:548358:256:4294967295] 192.168.202.128@tcp LPNI seq info [548281:548281:8:4294967295] 00000400:00000200:0.0:1788575723.650646:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575723.650651:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575723.650655:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575723.650657:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575723.650660:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575723.650661:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575723.650682:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575723.650683:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575723.650686:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575723.650689:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575723.650692:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575723.650694:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575723.650696:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575723.650699:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516858368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650707:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575723.650710:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516858368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650724:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.650725:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575723.650728:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575723.650730:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575723.650732:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575723.650734:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311083546 00000100:00000001:0.0:1788575723.650735:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311087342 00000100:00000001:0.0:1788575723.650736:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311083546 00000100:00000001:0.0:1788575723.650738:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311087342 00000100:00000001:0.0:1788575723.650739:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575723.650740:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.650741:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575723.650744:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575723/real 1788575723] req@ffff8fc4901a7480 x1875440516858368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575723.650750:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575723.650751:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575723.650753:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575723.650754:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575723.650756:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575723.650758:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575723.650761:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516858368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575723.650769:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516858368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650775:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575723.650776:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.650778:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575723.650779:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575723.650782:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516858368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650793:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575723.650797:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516858368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650804:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575723.650806:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.650822:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575723.650823:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.650824:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575723.650826:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575723.650828:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575723.650831:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575723.650832:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575723.650835:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516858368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575723.650840:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516858368:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575723.650842:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575723.650844:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516858496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650848:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575723.650849:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575723.650851:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575723.650852:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575723.650854:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575723.650856:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516858496:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575723.650857:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575723.650858:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575723.650863:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575723.650864:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575723.650866:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575723.650868:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575723.650871:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575723.650873:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575723.650875:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575723.650877:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516858496, portal 4 00000100:00000001:0.0:1788575723.650878:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575723.650879:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575723.650883:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516858496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650887:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575723.650889:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575723.650890:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575723.650891:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516858496, offset 0 00000400:00000200:0.0:1788575723.650894:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575723.650896:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575723.650898:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575723.650900:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575723.650902:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575723.650903:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575723.650904:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575723.650907:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548359:548359:256:4294967295] 192.168.202.128@tcp LPNI seq info [548282:548282:8:4294967295] 00000400:00000200:0.0:1788575723.650909:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575723.650915:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575723.650919:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575723.650922:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575723.650926:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575723.650929:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575723.650940:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575723.650947:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575723.650950:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575723.650952:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575723.650955:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575723.650956:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575723.650957:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575723.650960:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516858496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650965:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575723.650967:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516858496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.650971:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.650972:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575723.650974:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575723.650976:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575723.650978:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575723.650979:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311083687 00000100:00000001:0.0:1788575723.650981:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311087342 00000100:00000001:0.0:1788575723.650982:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311083687 00000100:00000001:0.0:1788575723.650984:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311087342 00000100:00000001:0.0:1788575723.650985:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575723.650986:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.650987:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575723.650989:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575723/real 1788575723] req@ffff8fc4901a4700 x1875440516858496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575723.650994:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575723.650995:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575723.650996:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575723.650997:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575723.650998:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575723.651003:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575723.651005:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516858496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575723.651010:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516858496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651013:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575723.651014:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651015:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575723.651016:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575723.651018:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516858496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651021:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575723.651024:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516858496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651028:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575723.651029:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651030:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575723.651031:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651032:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575723.651033:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575723.651034:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575723.651036:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575723.651037:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575723.651039:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516858496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575723.651043:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516858496:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575723.651044:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651046:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575723.651050:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516858240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651054:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575723.651055:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575723.651056:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575723.651058:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575723.651059:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575723.651060:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575723.651061:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575723.651062:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575723.651064:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575723.651065:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575723.651066:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575723.651070:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575723.651071:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651072:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575723.651074:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516858368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651078:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575723.651079:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575723.651080:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575723.651081:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575723.651081:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575723.651082:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575723.651083:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575723.651084:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575723.651085:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575723.651085:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575723.651086:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575723.651088:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575723.651089:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651089:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575723.651091:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516858496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651094:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575723.651097:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575723.651098:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575723.651099:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575723.651100:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575723.651101:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575723.651102:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575723.651103:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575723.651104:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575723.651105:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575723.651105:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575723.651107:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575723.651108:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651109:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651111:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575723.651112:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.651113:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575723.651114:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575723.651115:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575723.651118:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516858624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651123:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575723.651125:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575723.651126:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575723.651128:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575723.651130:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575723.651133:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516858624:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575723.651135:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575723.651137:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575723.651139:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575723.651141:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575723.651143:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575723.651145:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575723.651148:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575723.651151:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575723.651154:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575723.651157:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516858624, portal 4 00000100:00000001:0.0:1788575723.651163:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575723.651164:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575723.651168:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516858624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651172:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575723.651174:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575723.651176:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575723.651177:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516858624, offset 0 00000400:00000200:0.0:1788575723.651180:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575723.651182:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575723.651184:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575723.651186:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575723.651188:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575723.651189:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575723.651191:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575723.651193:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548360:548360:256:4294967295] 192.168.202.128@tcp LPNI seq info [548283:548283:8:4294967295] 00000400:00000200:0.0:1788575723.651195:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575723.651199:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575723.651201:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575723.651203:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575723.651206:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575723.651207:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788575723.651217:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575723.651218:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575723.651220:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575723.651223:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575723.651225:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575723.651226:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575723.651228:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575723.651230:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516858624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651241:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575723.651243:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516858624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575748 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651247:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.651248:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575723.651250:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575723.651252:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575723.651254:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575723.651255:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311083689 00000100:00000001:0.0:1788575723.651257:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311087342 00000100:00000001:0.0:1788575723.651258:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311083689 00000100:00000001:0.0:1788575723.651259:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311087342 00000100:00000001:0.0:1788575723.651260:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575723.651261:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.651263:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575723.651264:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575723/real 1788575723] req@ffff8fc4901a5880 x1875440516858624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575723.651269:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575723.651270:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575723.651271:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575723.651272:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575723.651273:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575723.651275:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575723.651276:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516858624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575723.651282:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516858624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651285:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575723.651289:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651290:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575723.651291:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575723.651294:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516858624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651297:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575723.651299:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516858624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651303:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575723.651304:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651305:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575723.651306:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651308:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575723.651309:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575723.651310:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575723.651312:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575723.651313:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575723.651315:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516858624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575723.651319:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516858624:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575723.651321:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651322:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575723.651324:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516858624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575748 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575723.651327:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575723.651328:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575723.651330:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575723.651331:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575723.651332:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575723.651335:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575723.651336:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575723.651337:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575723.651339:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575723.651341:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575723.651342:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575723.651345:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575723.651346:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651348:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575723.651351:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575723.651353:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575723.651354:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575723.651356:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575724.160982:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575724.160992:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575724.160997:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575724.161006:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575724.161013:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575724.161019:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575724.161045:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575724.161050:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575724.161054:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575725.184962:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575725.184972:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575725.184977:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575725.184987:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575725.184995:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575725.185004:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575725.185009:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575725.185014:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575725.185019:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575726.208936:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575726.208946:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575726.208950:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575726.208958:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575726.208967:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575726.208975:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575726.208982:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575726.209008:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575726.209012:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575726.209085:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575726.209091:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575726.209093:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575726.209097:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575726.209099:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575726.209103:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575726.209107:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575726.209109:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575726.209112:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575726.913065:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575726.913071:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575726.913075:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575727.232926:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575727.232934:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575727.232939:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575727.232946:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575727.232952:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575727.232957:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575727.233849:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575727.233851:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575727.233853:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575727.488868:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311087342, at 4311087360 00000100:00000010:0.0:1788575727.488882:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575727.936976:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575727.936980:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575727.936984:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575728.256939:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575728.256944:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575728.256948:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575728.256955:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575728.256960:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575728.256965:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575728.256988:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575728.256991:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575728.256994:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575728.768988:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575728.769008:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575728.769011:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575728.769016:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575728.769022:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575728.769030:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16416 00000400:00000200:0.0:1788575728.769042:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575728.769049:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575728.769052:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575728.769055:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769059:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575728.769062:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769064:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575728.769069:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575728.769072:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575728.769075:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575728.769078:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575728.769081:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575728.769083:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575728.769086:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769088:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575728.769090:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769093:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575728.769096:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575728.769100:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769102:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575728.769105:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575728.769109:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575728.769112:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769115:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575728.769117:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575728.769164:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575728.769169:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575728.769174:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575728.769176:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769179:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575728.769183:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575728.769189:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575728.769199:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575728.769208:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575728.769220:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516858752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575728.769231:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575728.769235:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575728.769239:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516858752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575728.769258:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575728.769261:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575728.769264:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575728.769265:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575728.769266:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575728.769268:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575728.769269:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16416 00000400:00000200:0.0:1788575728.769272:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575728.769274:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575728.769276:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575728.769278:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769280:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575728.769281:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769283:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575728.769285:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575728.769291:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575728.769303:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575728.769308:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575728.769312:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575728.769315:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575728.769324:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769330:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575728.769333:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769337:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575728.769350:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575728.769360:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769366:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575728.769369:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575728.769378:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575728.769383:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769387:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575728.769393:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575728.769398:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769403:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575728.769407:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575728.769411:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575728.769424:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575728.769426:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769427:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575728.769429:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575728.769431:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575728.769437:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575728.769441:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575728.769448:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516858880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575728.769460:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575728.769465:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575728.769473:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516858880/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575728.769492:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575728.769497:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575728.769502:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575728.769505:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575728.769507:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575728.769512:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575728.769517:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16416 00000400:00000200:0.0:1788575728.769522:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575728.769525:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575728.769526:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575728.769531:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769533:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575728.769535:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769537:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575728.769540:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575728.769543:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575728.769546:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575728.769548:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575728.769549:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575728.769552:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575728.769555:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769557:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575728.769560:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769563:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575728.769566:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575728.769570:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769573:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575728.769584:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575728.769588:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575728.769593:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769594:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575728.769596:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575728.769598:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769601:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575728.769605:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575728.769608:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575728.769613:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575728.769615:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769617:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575728.769619:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575728.769622:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575728.769628:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575728.769632:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575728.769638:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516859008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575728.769648:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575728.769651:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575728.769657:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516859008/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575728.769669:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575728.769673:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575728.769679:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575728.769682:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575728.769684:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575728.769687:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575728.769690:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16416 00000400:00000200:0.0:1788575728.769693:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575728.769699:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575728.769702:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575728.769710:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769712:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575728.769717:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769722:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575728.769723:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575728.769724:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575728.769726:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575728.769729:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575728.769732:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575728.769734:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575728.769741:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769745:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575728.769747:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769752:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575728.769754:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575728.769756:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769760:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575728.769762:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575728.769764:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575728.769768:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769770:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575728.769772:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575728.769776:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769780:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575728.769782:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575728.769785:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575728.769788:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575728.769789:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769791:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575728.769793:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575728.769795:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575728.769799:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575728.769832:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575728.769839:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516859136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575728.769848:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575728.769850:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575728.769854:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516859136/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575728.769864:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575728.769869:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575728.769872:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575728.769874:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575728.769877:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575728.769880:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575728.769883:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16416 00000400:00000200:0.0:1788575728.769888:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575728.769892:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575728.769896:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575728.769901:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769905:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575728.769908:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769911:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575728.769914:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575728.769915:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575728.769916:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575728.769918:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575728.769921:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575728.769924:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575728.769927:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769928:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575728.769935:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769939:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575728.769944:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575728.769947:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575728.769949:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575728.769951:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575728.769954:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575728.769958:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769961:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575728.769964:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575728.769970:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769972:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575728.769975:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575728.769979:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575728.769983:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575728.769986:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575728.769988:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575728.769990:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575728.769993:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575728.769996:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575728.769998:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575728.770003:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516859264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575728.770011:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575728.770014:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575728.770019:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516859264/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575728.770030:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575728.770036:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16426) 00000100:00000001:0.0:1788575729.281096:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575729.281103:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575729.281106:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575729.281117:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575729.281122:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575729.281124:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575729.281126:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575729.281127:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575729.281131:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575730.306214:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575730.306218:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575730.306220:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575730.306224:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575730.306226:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575730.306229:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575730.311445:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575730.311451:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575730.311535:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575731.329012:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575731.329018:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575731.329022:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575731.329028:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575731.329032:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575731.329035:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575731.329047:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575731.329050:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575731.329053:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575732.032992:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575732.032996:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575732.032999:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575732.353133:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575732.353136:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575732.353137:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575732.353139:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575732.353141:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575732.353144:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575732.353314:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575732.353316:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575732.353317:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575732.608921:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311092461, at 4311092480 00000100:00000010:0.0:1788575732.608936:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfe40. 00000020:00000001:0.0:1788575733.057023:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575733.057027:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575733.057031:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575733.376953:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575733.376969:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575733.376976:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575733.376991:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575733.377002:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575733.377010:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.377398:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575733.377400:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.377403:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.890043:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575733.890050:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575733.890054:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575733.890064:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516859392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.890083:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575733.890086:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575733.890092:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575733.890096:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575733.890102:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575733.890109:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516859392:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575733.890118:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575733.890124:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575733.890127:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575733.890131:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575733.890140:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575733.890143:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575733.890153:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575733.890159:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575733.890200:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575733.890218:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516859392, portal 25 00000100:00000001:0.0:1788575733.890223:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575733.890226:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575733.890258:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516859392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.890277:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575733.890285:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575733.890290:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575733.890297:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516859392, offset 0 00000400:00000200:0.0:1788575733.890305:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575733.890312:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575733.890317:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575733.890326:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575733.890330:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575733.890333:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575733.890336:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575733.890339:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548366:548366:256:4294967295] 192.168.202.128@tcp LPNI seq info [548289:548289:8:4294967295] 00000400:00000200:0.0:1788575733.890344:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575733.890353:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575733.890359:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575733.890366:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575733.890378:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575733.890385:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575733.890412:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575733.890415:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575733.890423:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575733.890432:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575733.890439:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575733.890444:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575733.890451:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575733.890459:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516859392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.890475:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575733.890492:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516859392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.890506:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.890509:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575733.890516:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575733.890523:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575733.890528:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311097582 00000100:00000010:0.0:1788575733.890535:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575733.890544:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575733.890552:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311097582 00000100:00000010:0.0:1788575733.890558:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50180. 00000100:00000001:0.0:1788575733.890563:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575733.890570:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575733.890575:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.890580:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575733.890588:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575733/real 1788575733] req@ffff8fc4901a7800 x1875440516859392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575733.890606:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575733.890609:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575733.890613:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575733.890619:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575733.890624:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575733.890628:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575733.890635:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516859392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575733.890655:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516859392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.890670:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575733.890673:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.890678:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575733.890682:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575733.890688:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516859392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.890707:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575733.890714:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516859392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.890727:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575733.890733:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.890737:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575733.890741:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.890746:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575733.890749:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575733.890754:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575733.890762:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575733.890767:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575733.890776:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516859392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575733.890790:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516859392:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575733.890798:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.890803:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575733.890850:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516859392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.890864:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575733.890868:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575733.890873:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575733.890879:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575733.890883:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575733.890887:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575733.890893:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575733.890897:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575733.890904:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575733.890912:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575733.890916:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575733.890923:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575733.890931:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.890936:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.890945:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575733.890948:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.890952:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575733.890955:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575733.890959:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575733.890965:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516859520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.890984:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575733.890987:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575733.890991:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575733.890995:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575733.891000:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575733.891021:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516859520:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575733.891026:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575733.891029:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575733.891031:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575733.891033:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575733.891036:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575733.891039:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575733.891043:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575733.891046:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575733.891049:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575733.891053:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516859520, portal 10 00000100:00000001:0.0:1788575733.891054:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575733.891056:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575733.891062:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516859520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891068:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575733.891070:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575733.891078:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575733.891080:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516859520, offset 0 00000400:00000200:0.0:1788575733.891084:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575733.891088:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575733.891091:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575733.891094:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575733.891096:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575733.891098:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575733.891101:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575733.891104:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548367:548367:256:4294967295] 192.168.202.128@tcp LPNI seq info [548290:548290:8:4294967295] 00000400:00000200:0.0:1788575733.891107:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575733.891113:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575733.891117:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575733.891120:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575733.891124:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575733.891126:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575733.891137:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575733.891138:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575733.891141:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575733.891144:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575733.891148:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575733.891150:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575733.891152:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575733.891156:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516859520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891190:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575733.891194:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516859520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891200:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.891201:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575733.891210:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575733.891214:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575733.891218:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575733.891220:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311093780 00000100:00000001:0.0:1788575733.891222:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311097582 00000100:00000001:0.0:1788575733.891224:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311093780 00000100:00000001:0.0:1788575733.891226:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311097582 00000100:00000001:0.0:1788575733.891229:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575733.891231:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.891233:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575733.891236:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575733/real 1788575733] req@ffff8fc4901a5c00 x1875440516859520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575733.891244:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575733.891245:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575733.891247:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575733.891249:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575733.891251:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575733.891253:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575733.891256:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516859520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575733.891263:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516859520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891269:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575733.891270:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891272:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575733.891274:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575733.891277:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516859520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891283:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575733.891288:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516859520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891298:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575733.891300:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891302:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575733.891303:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891305:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575733.891307:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575733.891310:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575733.891313:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575733.891315:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575733.891319:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516859520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575733.891327:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516859520:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575733.891331:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575733.891333:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516859648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891339:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575733.891340:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575733.891342:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575733.891344:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575733.891346:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575733.891349:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516859648:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575733.891352:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575733.891354:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575733.891355:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575733.891357:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575733.891360:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575733.891362:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575733.891370:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575733.891372:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575733.891376:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575733.891379:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516859648, portal 10 00000100:00000001:0.0:1788575733.891381:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575733.891382:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575733.891388:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516859648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891393:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575733.891395:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575733.891397:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575733.891399:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516859648, offset 0 00000400:00000200:0.0:1788575733.891402:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575733.891406:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575733.891409:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575733.891411:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575733.891414:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575733.891416:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575733.891418:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575733.891420:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548368:548368:256:4294967295] 192.168.202.128@tcp LPNI seq info [548291:548291:8:4294967295] 00000400:00000200:0.0:1788575733.891424:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575733.891430:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575733.891435:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575733.891438:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575733.891442:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575733.891445:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575733.891454:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575733.891456:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575733.891458:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575733.891483:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575733.891488:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575733.891496:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575733.891500:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575733.891503:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516859648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891511:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575733.891514:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516859648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891520:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.891522:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575733.891526:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575733.891529:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575733.891532:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575733.891534:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311093787 00000100:00000001:0.0:1788575733.891536:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311097582 00000100:00000001:0.0:1788575733.891538:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311093787 00000100:00000001:0.0:1788575733.891540:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311097582 00000100:00000001:0.0:1788575733.891542:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575733.891544:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.891546:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575733.891549:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575733/real 1788575733] req@ffff8fc4901a5180 x1875440516859648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575733.891557:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575733.891558:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575733.891560:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575733.891562:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575733.891563:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575733.891565:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575733.891568:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516859648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575733.891575:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516859648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891586:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575733.891587:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891589:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575733.891590:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575733.891593:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516859648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891599:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575733.891602:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516859648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891608:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575733.891610:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891611:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575733.891612:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891614:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575733.891616:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575733.891619:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575733.891622:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575733.891623:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575733.891627:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516859648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575733.891634:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516859648:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575733.891638:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575733.891641:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516859776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891647:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575733.891648:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575733.891655:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575733.891657:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575733.891658:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575733.891661:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516859776:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575733.891663:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575733.891664:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575733.891665:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575733.891667:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575733.891669:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575733.891670:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575733.891672:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575733.891674:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575733.891676:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575733.891678:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516859776, portal 4 00000100:00000001:0.0:1788575733.891679:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575733.891680:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575733.891684:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516859776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891688:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575733.891689:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575733.891690:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575733.891692:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516859776, offset 0 00000400:00000200:0.0:1788575733.891694:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575733.891696:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575733.891698:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575733.891700:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575733.891702:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575733.891703:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575733.891705:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575733.891707:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548369:548369:256:4294967295] 192.168.202.128@tcp LPNI seq info [548292:548292:8:4294967295] 00000400:00000200:0.0:1788575733.891709:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575733.891713:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575733.891719:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575733.891721:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575733.891723:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575733.891725:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575733.891732:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575733.891733:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575733.891735:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575733.891737:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575733.891739:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575733.891740:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575733.891742:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575733.891744:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516859776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891748:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575733.891751:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516859776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891757:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.891759:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575733.891762:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575733.891764:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575733.891768:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575733.891770:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311093927 00000100:00000001:0.0:1788575733.891772:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311097582 00000100:00000001:0.0:1788575733.891774:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311093927 00000100:00000001:0.0:1788575733.891776:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311097582 00000100:00000001:0.0:1788575733.891778:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575733.891779:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.891781:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575733.891785:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575733/real 1788575733] req@ffff8fc4901a7100 x1875440516859776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575733.891797:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575733.891798:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575733.891800:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575733.891803:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575733.891805:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575733.891826:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575733.891830:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516859776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575733.891836:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516859776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891840:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575733.891841:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891842:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575733.891843:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575733.891845:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516859776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891849:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575733.891851:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516859776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891856:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575733.891857:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891858:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575733.891859:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891860:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575733.891861:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575733.891862:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575733.891864:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575733.891865:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575733.891871:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516859776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575733.891879:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516859776:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575733.891883:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891885:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575733.891888:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516859520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891894:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575733.891896:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575733.891898:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575733.891901:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575733.891902:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575733.891904:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575733.891906:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575733.891908:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575733.891910:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575733.891912:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575733.891913:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575733.891918:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575733.891919:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891920:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575733.891924:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516859648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891930:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575733.891931:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575733.891933:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575733.891935:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575733.891936:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575733.891937:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575733.891939:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575733.891940:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575733.891943:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575733.891950:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575733.891951:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575733.891954:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575733.891955:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891957:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575733.891961:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516859776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891967:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575733.891969:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575733.891970:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575733.891972:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575733.891972:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575733.891974:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575733.891975:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575733.891976:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575733.891977:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575733.891978:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575733.891979:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575733.891981:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575733.891981:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891982:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.891985:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575733.891986:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.891988:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575733.891989:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575733.891990:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575733.891992:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516859904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.891997:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575733.891998:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575733.891999:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575733.892000:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575733.892002:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575733.892004:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516859904:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575733.892005:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575733.892011:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575733.892012:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575733.892013:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575733.892015:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575733.892016:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575733.892019:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575733.892021:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575733.892023:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575733.892025:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516859904, portal 4 00000100:00000001:0.0:1788575733.892026:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575733.892027:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575733.892031:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516859904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.892035:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575733.892037:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575733.892038:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575733.892039:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516859904, offset 0 00000400:00000200:0.0:1788575733.892042:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575733.892044:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575733.892046:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575733.892048:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575733.892049:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575733.892051:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575733.892053:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575733.892055:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548370:548370:256:4294967295] 192.168.202.128@tcp LPNI seq info [548293:548293:8:4294967295] 00000400:00000200:0.0:1788575733.892057:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575733.892061:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575733.892065:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575733.892068:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575733.892072:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575733.892074:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575733.892084:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575733.892091:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575733.892094:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575733.892097:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575733.892101:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575733.892102:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575733.892105:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575733.892108:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516859904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.892116:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575733.892120:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516859904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575758 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.892127:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.892128:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575733.892132:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575733.892135:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575733.892140:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575733.892142:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311093930 00000100:00000001:0.0:1788575733.892145:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311097582 00000100:00000001:0.0:1788575733.892147:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311093930 00000100:00000001:0.0:1788575733.892150:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311097582 00000100:00000001:0.0:1788575733.892152:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575733.892153:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.892155:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575733.892159:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575733/real 1788575733] req@ffff8fc4901a4a80 x1875440516859904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575733.892180:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575733.892182:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575733.892184:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575733.892186:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575733.892188:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575733.892194:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575733.892197:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516859904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575733.892202:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516859904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.892207:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575733.892207:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.892208:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575733.892209:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575733.892211:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516859904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.892216:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575733.892218:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516859904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.892222:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575733.892223:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.892224:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575733.892224:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.892225:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575733.892227:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575733.892228:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575733.892230:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575733.892231:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575733.892233:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516859904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575733.892237:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516859904:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575733.892239:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.892241:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575733.892256:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516859904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575758 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575733.892260:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575733.892261:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575733.892262:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575733.892264:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575733.892265:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575733.892266:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575733.892267:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575733.892268:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575733.892270:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575733.892271:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575733.892272:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575733.892274:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575733.892275:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.892276:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575733.892279:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575733.892280:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575733.892281:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575733.892282:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575734.400915:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575734.400924:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575734.400929:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575734.400937:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575734.400944:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575734.400951:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575734.400959:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575734.400964:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575734.400971:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575734.401018:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575734.401022:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575734.401023:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575734.401026:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575734.401029:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575734.401031:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575734.401032:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575734.401034:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575734.401056:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575736.452052:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575736.452055:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575736.452056:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575736.452058:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575736.452060:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575736.452062:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575736.452064:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575736.452065:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575736.452067:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575737.153053:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575737.153056:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575737.153058:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575737.472982:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575737.473002:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575737.473006:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575737.473012:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575737.473018:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575737.473024:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575737.473030:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575737.473034:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575737.473043:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575737.473084:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575737.473091:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575737.473092:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575737.473095:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575737.473101:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575737.473104:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575737.473107:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575737.473110:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575737.473112:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575737.728945:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311097582, at 4311097600 00000100:00000010:0.0:1788575737.728969:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575738.178021:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575738.178025:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575738.178029:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575738.497845:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575738.497873:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575738.497876:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575738.497884:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575738.497893:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575738.497898:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575738.497908:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575738.497910:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575738.497919:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.009184:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575739.009190:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575739.009194:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575739.009204:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6680 x1875440516860032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.009229:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575739.009233:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575739.009239:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575739.009243:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575739.009249:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575739.009255:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516860032:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575739.009264:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575739.009270:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575739.009274:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575739.009281:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575739.009288:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575739.009292:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575739.009301:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575739.009307:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575739.009323:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575739.009332:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516860032, portal 25 00000100:00000001:0.0:1788575739.009335:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575739.009339:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605330560 : -123400104221056 : ffff8fc4ac9f6680) 00000100:00000040:0.0:1788575739.009349:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6680 x1875440516860032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.009367:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575739.009381:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575739.009387:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575739.009393:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516860032, offset 0 00000400:00000200:0.0:1788575739.009403:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575739.009413:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575739.009420:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575739.009425:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575739.009429:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575739.009432:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575739.009435:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575739.009439:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548371:548371:256:4294967295] 192.168.202.128@tcp LPNI seq info [548294:548294:8:4294967295] 00000400:00000200:0.0:1788575739.009444:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575739.009453:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575739.009462:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575739.009470:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575739.009482:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575739.009489:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575739.009526:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575739.009531:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575739.009539:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575739.009547:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575739.009556:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575739.009561:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575739.009569:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575739.009576:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6680 x1875440516860032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.009594:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575739.009609:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6680 x1875440516860032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.009622:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.009627:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575739.009640:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575739.009650:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575739.009656:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311102701 00000100:00000010:0.0:1788575739.009668:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575739.009677:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575739.009685:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311102701 00000100:00000010:0.0:1788575739.009691:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b509c0. 00000100:00000001:0.0:1788575739.009696:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575739.009701:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575739.009704:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.009709:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575739.009716:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575739/real 1788575739] req@ffff8fc4ac9f6680 x1875440516860032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575739.009737:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575739.009741:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575739.009746:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575739.009750:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575739.009752:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575739.009755:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575739.009760:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6680 x1875440516860032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575739.009780:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6680 x1875440516860032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.009794:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575739.009797:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.009802:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575739.009806:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575739.009831:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6680 x1875440516860032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.009838:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575739.009846:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6680 x1875440516860032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.009868:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575739.009874:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.009878:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575739.009881:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.009886:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575739.009890:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575739.009896:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575739.009901:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575739.009906:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575739.009911:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6680 x1875440516860032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575739.009938:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516860032:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575739.009947:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.009951:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575739.009956:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6680 x1875440516860032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.009969:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575739.009973:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575739.009977:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575739.009983:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575739.009987:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575739.009999:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575739.010005:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575739.010010:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36400. 02000000:00000001:0.0:1788575739.010017:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575739.010021:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575739.010026:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6680. 00000100:00000001:0.0:1788575739.010032:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575739.010035:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.010039:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.010053:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575739.010057:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.010061:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575739.010064:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575739.010067:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575739.010076:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4000 x1875440516860160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010090:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575739.010093:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575739.010097:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575739.010098:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575739.010103:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575739.010107:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516860160:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575739.010111:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575739.010114:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575739.010116:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575739.010119:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575739.010123:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575739.010127:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575739.010133:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575739.010138:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575739.010143:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575739.010148:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516860160, portal 10 00000100:00000001:0.0:1788575739.010153:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575739.010156:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605320704 : -123400104230912 : ffff8fc4ac9f4000) 00000100:00000040:0.0:1788575739.010165:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4000 x1875440516860160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010176:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575739.010181:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575739.010185:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575739.010189:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516860160, offset 0 00000400:00000200:0.0:1788575739.010198:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575739.010204:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575739.010216:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575739.010222:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575739.010227:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575739.010232:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575739.010236:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575739.010241:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548372:548372:256:4294967295] 192.168.202.128@tcp LPNI seq info [548295:548295:8:4294967295] 00000400:00000200:0.0:1788575739.010248:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575739.010254:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575739.010260:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575739.010263:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575739.010267:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575739.010270:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575739.010280:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575739.010281:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575739.010284:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575739.010288:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575739.010290:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575739.010292:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575739.010295:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575739.010298:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4000 x1875440516860160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010305:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575739.010307:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4000 x1875440516860160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010313:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.010315:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575739.010317:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575739.010321:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575739.010323:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575739.010325:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311098899 00000100:00000001:0.0:1788575739.010332:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311102701 00000100:00000001:0.0:1788575739.010334:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311098899 00000100:00000001:0.0:1788575739.010336:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311102701 00000100:00000001:0.0:1788575739.010338:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575739.010340:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.010342:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575739.010345:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575739/real 1788575739] req@ffff8fc4ac9f4000 x1875440516860160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575739.010353:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575739.010354:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575739.010355:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575739.010357:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575739.010359:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575739.010361:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575739.010364:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4000 x1875440516860160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575739.010370:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4000 x1875440516860160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010377:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575739.010378:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.010379:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575739.010381:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575739.010383:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4000 x1875440516860160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010389:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575739.010393:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4000 x1875440516860160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010399:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575739.010401:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.010406:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575739.010408:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.010409:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575739.010411:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575739.010414:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575739.010417:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575739.010419:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575739.010423:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4000 x1875440516860160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575739.010431:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516860160:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575739.010434:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575739.010437:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7100 x1875440516860288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010443:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575739.010444:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575739.010446:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575739.010448:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575739.010450:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575739.010452:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516860288:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575739.010455:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575739.010457:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575739.010458:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575739.010460:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575739.010463:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575739.010465:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575739.010469:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575739.010471:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575739.010474:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575739.010477:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516860288, portal 10 00000100:00000001:0.0:1788575739.010479:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575739.010487:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605333248 : -123400104218368 : ffff8fc4ac9f7100) 00000100:00000040:0.0:1788575739.010493:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7100 x1875440516860288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010500:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575739.010502:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575739.010504:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575739.010506:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516860288, offset 0 00000400:00000200:0.0:1788575739.010510:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575739.010514:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575739.010517:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575739.010520:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575739.010522:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575739.010524:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575739.010526:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575739.010529:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548373:548373:256:4294967295] 192.168.202.128@tcp LPNI seq info [548296:548296:8:4294967295] 00000400:00000200:0.0:1788575739.010533:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575739.010538:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575739.010542:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575739.010545:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575739.010549:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575739.010551:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575739.010560:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575739.010562:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575739.010564:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575739.010567:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575739.010570:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575739.010571:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575739.010574:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575739.010577:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7100 x1875440516860288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010592:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575739.010596:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7100 x1875440516860288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010601:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.010603:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575739.010606:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575739.010609:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575739.010612:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575739.010614:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311098906 00000100:00000001:0.0:1788575739.010617:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311102701 00000100:00000001:0.0:1788575739.010619:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311098906 00000100:00000001:0.0:1788575739.010621:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311102701 00000100:00000001:0.0:1788575739.010623:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575739.010624:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.010626:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575739.010629:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575739/real 1788575739] req@ffff8fc4ac9f7100 x1875440516860288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575739.010636:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575739.010637:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575739.010639:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575739.010641:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575739.010642:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575739.010644:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575739.010647:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7100 x1875440516860288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575739.010654:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7100 x1875440516860288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010660:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575739.010665:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.010667:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575739.010668:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575739.010671:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7100 x1875440516860288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010678:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575739.010681:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7100 x1875440516860288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010688:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575739.010690:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.010691:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575739.010692:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.010694:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575739.010697:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575739.010699:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575739.010702:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575739.010704:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575739.010708:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7100 x1875440516860288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575739.010715:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516860288:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575739.010719:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575739.010722:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516860416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010728:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575739.010730:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575739.010732:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575739.010733:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575739.010735:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575739.010744:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516860416:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575739.010747:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575739.010749:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575739.010751:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575739.010753:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575739.010756:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575739.010757:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575739.010761:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575739.010763:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575739.010767:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575739.010770:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516860416, portal 4 00000100:00000001:0.0:1788575739.010771:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575739.010773:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788575739.010778:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516860416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010784:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575739.010786:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575739.010788:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575739.010790:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516860416, offset 0 00000400:00000200:0.0:1788575739.010794:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575739.010797:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575739.010801:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575739.010804:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575739.010806:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575739.010825:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575739.010828:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575739.010831:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548374:548374:256:4294967295] 192.168.202.128@tcp LPNI seq info [548297:548297:8:4294967295] 00000400:00000200:0.0:1788575739.010835:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575739.010841:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575739.010846:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575739.010849:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575739.010858:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575739.010861:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575739.010870:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575739.010872:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575739.010875:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575739.010878:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575739.010881:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575739.010883:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575739.010886:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575739.010890:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516860416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010897:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575739.010900:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516860416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010907:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.010909:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575739.010912:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575739.010927:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575739.010931:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575739.010933:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311099047 00000100:00000001:0.0:1788575739.010936:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311102701 00000100:00000001:0.0:1788575739.010938:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311099047 00000100:00000001:0.0:1788575739.010940:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311102701 00000100:00000001:0.0:1788575739.010942:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575739.010944:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.010946:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575739.010950:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575739/real 1788575739] req@ffff8fc4ac9f6a00 x1875440516860416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575739.010958:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575739.010965:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575739.010967:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575739.010969:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575739.010971:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575739.010974:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575739.010978:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516860416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575739.010986:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516860416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.010993:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575739.010994:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.010996:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575739.010997:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575739.010999:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516860416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011004:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575739.011006:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516860416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011009:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575739.011011:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011011:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575739.011012:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011013:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575739.011014:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575739.011016:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575739.011017:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575739.011019:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575739.011021:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516860416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575739.011035:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516860416:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575739.011039:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011041:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575739.011044:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4000 x1875440516860160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011051:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575739.011053:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575739.011055:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575739.011057:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575739.011059:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575739.011061:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575739.011064:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575739.011066:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35c00. 02000000:00000001:0.0:1788575739.011069:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575739.011070:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575739.011072:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4000. 00000100:00000001:0.0:1788575739.011075:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575739.011076:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011078:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575739.011082:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7100 x1875440516860288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011088:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575739.011090:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575739.011091:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575739.011094:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575739.011096:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575739.011097:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575739.011099:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575739.011101:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34c00. 02000000:00000001:0.0:1788575739.011103:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575739.011104:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575739.011106:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7100. 00000100:00000001:0.0:1788575739.011114:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575739.011116:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011117:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575739.011121:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516860416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011127:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575739.011128:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575739.011130:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575739.011132:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575739.011133:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575739.011134:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575739.011136:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575739.011137:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35000. 02000000:00000001:0.0:1788575739.011140:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575739.011141:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575739.011142:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788575739.011144:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575739.011146:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011148:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011152:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575739.011153:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.011155:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575739.011157:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575739.011158:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575739.011162:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7800 x1875440516860544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011169:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575739.011170:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575739.011172:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575739.011174:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575739.011176:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575739.011179:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516860544:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575739.011183:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575739.011185:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575739.011186:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575739.011192:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575739.011195:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575739.011197:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575739.011201:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575739.011203:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575739.011206:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575739.011209:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516860544, portal 4 00000100:00000001:0.0:1788575739.011211:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575739.011212:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335040 : -123400104216576 : ffff8fc4ac9f7800) 00000100:00000040:0.0:1788575739.011217:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7800 x1875440516860544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011224:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575739.011226:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575739.011229:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575739.011231:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516860544, offset 0 00000400:00000200:0.0:1788575739.011235:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575739.011239:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575739.011243:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575739.011246:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575739.011248:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575739.011250:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575739.011253:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575739.011256:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548375:548375:256:4294967295] 192.168.202.128@tcp LPNI seq info [548298:548298:8:4294967295] 00000400:00000200:0.0:1788575739.011260:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575739.011267:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575739.011272:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575739.011275:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575739.011279:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575739.011281:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1788575739.011292:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575739.011294:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575739.011302:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575739.011306:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575739.011310:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575739.011311:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575739.011314:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575739.011317:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7800 x1875440516860544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011325:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575739.011329:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7800 x1875440516860544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575764 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011335:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.011337:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575739.011340:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575739.011343:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575739.011346:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575739.011348:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311099049 00000100:00000001:0.0:1788575739.011351:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311102701 00000100:00000001:0.0:1788575739.011353:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311099049 00000100:00000001:0.0:1788575739.011355:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311102701 00000100:00000001:0.0:1788575739.011358:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575739.011359:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.011362:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575739.011365:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575739/real 1788575739] req@ffff8fc4ac9f7800 x1875440516860544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575739.011373:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575739.011374:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575739.011376:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575739.011378:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575739.011380:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575739.011382:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575739.011390:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7800 x1875440516860544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575739.011398:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7800 x1875440516860544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011404:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575739.011406:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011408:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575739.011409:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575739.011413:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7800 x1875440516860544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011419:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575739.011423:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7800 x1875440516860544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011429:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575739.011432:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011433:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575739.011434:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011436:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575739.011438:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575739.011440:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575739.011443:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575739.011446:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575739.011449:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7800 x1875440516860544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575739.011456:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516860544:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575739.011460:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011462:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575739.011464:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7800 x1875440516860544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575764 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575739.011474:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575739.011476:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575739.011478:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575739.011480:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575739.011481:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575739.011482:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575739.011485:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575739.011486:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36400. 02000000:00000001:0.0:1788575739.011488:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575739.011490:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575739.011491:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7800. 00000100:00000001:0.0:1788575739.011493:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575739.011495:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011497:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.011500:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575739.011501:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.011503:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575739.011505:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.520931:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575739.520940:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575739.520944:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575739.520952:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575739.520959:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575739.520964:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.520971:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575739.520975:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.520978:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575739.521010:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575739.521014:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575739.521015:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575739.521017:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575739.521020:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575739.521022:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.521024:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575739.521027:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575739.521029:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575740.544951:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575740.544960:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575740.544963:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575740.544970:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575740.544976:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575740.544980:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575740.547790:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575740.547794:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575740.547798:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575741.568950:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575741.568957:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575741.568960:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575741.568963:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575741.568967:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575741.568969:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575741.568972:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575741.568974:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575741.568977:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575742.273028:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575742.273032:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575742.273035:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575742.592938:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575742.592948:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575742.592954:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575742.592960:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575742.592967:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575742.592974:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575742.592981:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575742.592986:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575742.592992:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575742.848925:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311102701, at 4311102720 00000100:00000010:0.0:1788575742.848942:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575743.296989:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575743.296991:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575743.296994:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575743.616947:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575743.616952:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575743.616969:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575743.616973:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575743.616976:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575743.616978:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575743.616980:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575743.616981:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575743.616984:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.129142:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575744.129148:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575744.129152:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575744.129161:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516860672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.129178:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575744.129182:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575744.129188:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575744.129193:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575744.129199:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575744.129207:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516860672:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575744.129215:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575744.129220:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575744.129223:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575744.129227:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575744.129233:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575744.129237:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575744.129245:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575744.129251:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575744.129260:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575744.129270:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516860672, portal 25 00000100:00000001:0.0:1788575744.129275:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575744.129277:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575744.129287:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516860672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.129320:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575744.129327:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575744.129347:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575744.129352:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516860672, offset 0 00000400:00000200:0.0:1788575744.129360:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575744.129376:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575744.129385:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575744.129392:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575744.129396:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575744.129400:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575744.129403:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575744.129408:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548376:548376:256:4294967295] 192.168.202.128@tcp LPNI seq info [548299:548299:8:4294967295] 00000400:00000200:0.0:1788575744.129414:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575744.129427:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575744.129438:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575744.129446:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575744.129456:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575744.129463:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575744.129486:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575744.129488:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575744.129495:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575744.129501:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575744.129508:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575744.129511:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575744.129517:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575744.129523:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516860672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.129535:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575744.129539:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516860672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.129549:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.129552:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575744.129564:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575744.129574:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575744.129579:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311107821 00000100:00000010:0.0:1788575744.129587:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575744.129596:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575744.129605:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311107821 00000100:00000010:0.0:1788575744.129613:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50540. 00000100:00000001:0.0:1788575744.129630:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575744.129637:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575744.129643:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.129648:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575744.129656:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575744/real 1788575744] req@ffff8fc4901a4a80 x1875440516860672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575744.129705:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575744.129708:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575744.129712:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575744.129717:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575744.129721:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575744.129726:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575744.129734:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516860672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575744.129757:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516860672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.129777:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575744.129780:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.129784:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575744.129789:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575744.129798:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516860672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.129827:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575744.129832:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516860672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.129847:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575744.129854:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.129858:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575744.129860:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.129863:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575744.129866:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575744.129870:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575744.129875:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575744.129878:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575744.129884:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516860672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575744.129902:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516860672:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575744.129908:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.129913:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575744.129917:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516860672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.129926:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575744.129929:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575744.129932:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575744.129936:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575744.129938:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575744.129941:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575744.129945:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575744.129948:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575744.129953:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575744.129956:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575744.129960:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575744.129967:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575744.129971:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.129975:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.129989:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575744.129992:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.129995:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575744.129997:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575744.129999:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575744.130006:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516860800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130014:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575744.130016:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575744.130018:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575744.130021:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575744.130023:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575744.130027:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516860800:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575744.130031:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575744.130033:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575744.130034:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575744.130036:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575744.130039:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575744.130041:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575744.130045:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575744.130048:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575744.130052:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575744.130056:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516860800, portal 10 00000100:00000001:0.0:1788575744.130058:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575744.130059:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575744.130065:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516860800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130072:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575744.130074:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575744.130076:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575744.130078:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516860800, offset 0 00000400:00000200:0.0:1788575744.130081:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575744.130085:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575744.130093:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575744.130096:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575744.130099:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575744.130101:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575744.130104:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575744.130107:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548377:548377:256:4294967295] 192.168.202.128@tcp LPNI seq info [548300:548300:8:4294967295] 00000400:00000200:0.0:1788575744.130111:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575744.130117:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575744.130121:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575744.130124:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575744.130128:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575744.130131:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575744.130145:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575744.130147:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575744.130150:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575744.130153:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575744.130156:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575744.130158:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575744.130161:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575744.130164:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516860800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130172:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575744.130174:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516860800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130181:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.130183:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575744.130186:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575744.130189:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575744.130193:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575744.130195:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311104019 00000100:00000001:0.0:1788575744.130201:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311107821 00000100:00000001:0.0:1788575744.130204:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311104019 00000100:00000001:0.0:1788575744.130206:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311107821 00000100:00000001:0.0:1788575744.130208:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575744.130210:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.130212:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575744.130216:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575744/real 1788575744] req@ffff8fc4901a7100 x1875440516860800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575744.130225:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575744.130226:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575744.130228:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575744.130230:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575744.130232:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575744.130234:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575744.130237:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516860800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575744.130246:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516860800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130253:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575744.130255:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.130257:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575744.130258:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575744.130262:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516860800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130269:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575744.130273:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516860800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130281:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575744.130283:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.130291:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575744.130293:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.130295:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575744.130297:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575744.130300:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575744.130303:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575744.130306:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575744.130310:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516860800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575744.130318:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516860800:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575744.130322:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575744.130325:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516860928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130332:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575744.130334:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575744.130336:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575744.130338:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575744.130340:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575744.130343:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516860928:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575744.130347:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575744.130348:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575744.130350:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575744.130352:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575744.130355:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575744.130357:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575744.130360:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575744.130363:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575744.130366:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575744.130369:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516860928, portal 10 00000100:00000001:0.0:1788575744.130372:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575744.130378:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575744.130384:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516860928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130390:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575744.130393:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575744.130395:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575744.130398:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516860928, offset 0 00000400:00000200:0.0:1788575744.130402:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575744.130405:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575744.130409:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575744.130412:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575744.130414:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575744.130416:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575744.130418:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575744.130421:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548378:548378:256:4294967295] 192.168.202.128@tcp LPNI seq info [548301:548301:8:4294967295] 00000400:00000200:0.0:1788575744.130425:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575744.130430:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575744.130435:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575744.130438:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575744.130442:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575744.130445:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575744.130456:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575744.130458:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575744.130461:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575744.130464:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575744.130468:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575744.130470:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575744.130472:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575744.130476:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516860928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130495:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575744.130498:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516860928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130505:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.130507:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575744.130510:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575744.130513:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575744.130517:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575744.130519:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311104026 00000100:00000001:0.0:1788575744.130522:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311107821 00000100:00000001:0.0:1788575744.130524:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311104026 00000100:00000001:0.0:1788575744.130526:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311107821 00000100:00000001:0.0:1788575744.130528:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575744.130530:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.130532:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575744.130536:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575744/real 1788575744] req@ffff8fc4901a5180 x1875440516860928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575744.130544:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575744.130545:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575744.130547:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575744.130549:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575744.130550:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575744.130552:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575744.130555:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516860928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575744.130564:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516860928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130570:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575744.130576:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.130578:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575744.130580:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575744.130584:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516860928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130592:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575744.130596:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516860928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130603:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575744.130605:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.130607:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575744.130608:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.130610:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575744.130612:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575744.130615:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575744.130618:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575744.130621:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575744.130624:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516860928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575744.130633:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516860928:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575744.130636:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575744.130640:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516861056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130647:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575744.130648:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575744.130650:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575744.130652:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575744.130654:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575744.130662:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516861056:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575744.130678:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575744.130679:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575744.130681:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575744.130682:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575744.130685:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575744.130687:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575744.130691:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575744.130693:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575744.130697:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575744.130700:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516861056, portal 4 00000100:00000001:0.0:1788575744.130702:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575744.130703:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575744.130709:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516861056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130715:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575744.130718:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575744.130720:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575744.130722:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516861056, offset 0 00000400:00000200:0.0:1788575744.130726:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575744.130729:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575744.130733:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575744.130736:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575744.130738:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575744.130740:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575744.130742:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575744.130745:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548379:548379:256:4294967295] 192.168.202.128@tcp LPNI seq info [548302:548302:8:4294967295] 00000400:00000200:0.0:1788575744.130750:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575744.130756:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575744.130760:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575744.130763:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575744.130772:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575744.130775:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575744.130786:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575744.130788:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575744.130791:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575744.130795:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575744.130799:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575744.130800:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575744.130803:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575744.130806:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516861056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130832:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575744.130835:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516861056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130842:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.130844:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575744.130847:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575744.130851:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575744.130854:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575744.130856:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311104167 00000100:00000001:0.0:1788575744.130859:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311107821 00000100:00000001:0.0:1788575744.130861:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311104167 00000100:00000001:0.0:1788575744.130864:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311107821 00000100:00000001:0.0:1788575744.130866:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575744.130868:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.130870:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575744.130873:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575744/real 1788575744] req@ffff8fc4901a5c00 x1875440516861056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575744.130881:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575744.130887:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575744.130889:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575744.130891:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575744.130893:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575744.130896:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575744.130899:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516861056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575744.130908:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516861056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130915:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575744.130916:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.130918:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575744.130920:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575744.130924:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516861056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130931:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575744.130935:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516861056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130942:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575744.130945:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.130946:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575744.130948:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.130949:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575744.130952:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575744.130954:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575744.130958:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575744.130960:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575744.130964:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516861056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575744.130984:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516861056:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575744.130988:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575744.130991:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516861184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.130998:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575744.130999:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575744.131002:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575744.131004:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575744.131006:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575744.131009:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516861184:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575744.131012:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575744.131014:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575744.131016:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575744.131017:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575744.131020:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575744.131022:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575744.131026:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575744.131029:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575744.131032:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575744.131041:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516861184, portal 4 00000100:00000001:0.0:1788575744.131043:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575744.131045:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575744.131050:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516861184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.131057:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575744.131060:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575744.131062:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575744.131064:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516861184, offset 0 00000400:00000200:0.0:1788575744.131068:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575744.131077:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575744.131081:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575744.131083:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575744.131086:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575744.131088:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575744.131091:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575744.131094:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548380:548380:256:4294967295] 192.168.202.128@tcp LPNI seq info [548303:548303:8:4294967295] 00000400:00000200:0.0:1788575744.131098:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575744.131104:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575744.131109:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575744.131112:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575744.131116:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575744.131119:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1788575744.131130:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575744.131132:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575744.131135:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575744.131138:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575744.131142:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575744.131144:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575744.131146:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575744.131150:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516861184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.131157:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575744.131161:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516861184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575769 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.131167:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.131169:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575744.131172:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575744.131174:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575744.131178:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575744.131184:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311104169 00000100:00000001:0.0:1788575744.131187:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311107821 00000100:00000001:0.0:1788575744.131190:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311104169 00000100:00000001:0.0:1788575744.131192:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311107821 00000100:00000001:0.0:1788575744.131194:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575744.131196:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.131198:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575744.131202:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575744/real 1788575744] req@ffff8fc4901a7800 x1875440516861184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575744.131211:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575744.131212:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575744.131214:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575744.131217:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575744.131219:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575744.131221:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575744.131224:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516861184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575744.131233:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516861184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.131240:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575744.131242:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.131244:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575744.131245:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575744.131249:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516861184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.131256:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575744.131259:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516861184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.131266:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575744.131273:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.131275:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575744.131276:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.131278:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575744.131281:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575744.131283:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575744.131287:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575744.131290:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575744.131293:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516861184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575744.131302:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516861184:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575744.131306:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.131309:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575744.131311:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516860800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.131318:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575744.131319:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575744.131321:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575744.131325:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575744.131326:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575744.131327:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575744.131329:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575744.131332:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575744.131335:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575744.131336:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575744.131338:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575744.131342:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575744.131343:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.131344:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575744.131348:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516860928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.131364:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575744.131366:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575744.131367:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575744.131370:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575744.131371:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575744.131372:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575744.131375:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575744.131376:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575744.131378:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575744.131380:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575744.131381:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575744.131385:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575744.131386:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.131388:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575744.131392:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516861056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.131398:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575744.131399:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575744.131401:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575744.131403:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575744.131405:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575744.131406:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575744.131408:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575744.131409:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575744.131411:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575744.131412:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575744.131414:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575744.131416:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575744.131417:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.131419:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575744.131422:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516861184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575769 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575744.131429:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575744.131430:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575744.131436:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575744.131439:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575744.131440:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575744.131441:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575744.131443:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575744.131444:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575744.131446:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575744.131447:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575744.131449:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575744.131451:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575744.131453:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.131455:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.131459:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575744.131460:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.131462:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575744.131463:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.640911:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575744.640919:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575744.640921:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575744.640926:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575744.640933:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575744.640938:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.640945:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575744.640949:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.640953:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575744.640971:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575744.640976:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575744.640980:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575744.640984:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575744.640988:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575744.640992:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.640997:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575744.641000:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575744.641003:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575745.665071:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575745.665075:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575745.665077:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575745.665080:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575745.665103:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575745.665105:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575745.665107:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575745.665108:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575745.665110:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575746.688931:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575746.688936:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575746.688940:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575746.688945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575746.688948:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575746.688952:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575746.688969:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575746.688972:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575746.688976:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575747.393007:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575747.393010:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575747.393013:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575747.712884:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575747.712892:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575747.712896:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575747.712903:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575747.712909:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575747.712914:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575747.712931:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575747.712934:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575747.712937:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575747.968906:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311107821, at 4311107840 00000100:00000010:0.0:1788575747.968923:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575748.417003:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575748.417007:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575748.417011:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575748.736951:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575748.736961:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575748.736966:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575748.736976:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575748.736990:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575748.737002:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575748.737037:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575748.737043:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575748.737050:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575748.737100:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575748.737108:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575748.737110:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575748.737113:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575748.737116:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575748.737118:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575748.737120:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575748.737122:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575748.737124:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.249222:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575749.249231:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575749.249238:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575749.249251:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516861312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.249278:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575749.249283:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575749.249291:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575749.249293:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575749.249296:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575749.249299:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516861312:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575749.249307:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575749.249313:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575749.249315:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575749.249318:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575749.249327:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575749.249335:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575749.249349:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575749.249354:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575749.249361:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575749.249371:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516861312, portal 25 00000100:00000001:0.0:1788575749.249378:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575749.249385:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788575749.249410:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516861312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.249454:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575749.249463:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575749.249467:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575749.249471:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516861312, offset 0 00000400:00000200:0.0:1788575749.249481:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575749.249493:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575749.249502:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575749.249511:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575749.249516:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575749.249523:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575749.249530:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575749.249539:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548381:548381:256:4294967295] 192.168.202.128@tcp LPNI seq info [548304:548304:8:4294967295] 00000400:00000200:0.0:1788575749.249559:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575749.249585:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575749.249594:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575749.249608:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575749.249621:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575749.249626:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575749.249654:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575749.249660:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575749.249668:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575749.249683:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575749.249690:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575749.249700:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575749.249706:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575749.249711:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516861312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.249722:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575749.249735:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516861312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.249749:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.249767:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575749.249780:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575749.249786:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575749.249788:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311112941 00000100:00000010:0.0:1788575749.249795:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575749.249803:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575749.249844:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311112942 00000100:00000010:0.0:1788575749.249848:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b502a0. 00000100:00000001:0.0:1788575749.249851:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575749.249855:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575749.249858:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.249866:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575749.249873:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575749/real 1788575749] req@ffff8fc4ac821c00 x1875440516861312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575749.249891:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575749.249894:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575749.249902:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575749.249905:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575749.249913:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575749.249917:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575749.249925:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516861312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575749.249938:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516861312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.249978:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575749.249979:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.249980:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575749.249983:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575749.250002:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516861312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.250032:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575749.250047:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516861312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.250075:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575749.250083:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.250090:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575749.250093:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.250101:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575749.250105:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575749.250117:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575749.250135:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575749.250146:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575749.250159:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516861312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575749.250187:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516861312:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575749.250206:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.250210:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575749.250214:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516861312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.250231:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575749.250235:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575749.250240:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575749.250249:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575749.250253:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575749.250259:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575749.250271:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575749.250282:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629800. 02000000:00000001:0.0:1788575749.250294:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575749.250306:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575749.250309:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788575749.250318:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575749.250320:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.250327:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.250341:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575749.250346:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.250356:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575749.250362:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575749.250366:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575749.250380:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516861440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.250434:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575749.250438:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575749.250448:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575749.250452:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575749.250463:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575749.250475:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516861440:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575749.250495:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575749.250505:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575749.250513:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575749.250521:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575749.250530:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575749.250541:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575749.250557:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575749.250564:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575749.250569:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575749.250573:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516861440, portal 10 00000100:00000001:0.0:1788575749.250577:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575749.250580:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788575749.250590:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516861440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.250598:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575749.250605:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575749.250613:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575749.250619:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516861440, offset 0 00000400:00000200:0.0:1788575749.250633:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575749.250646:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575749.250657:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575749.250665:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575749.250673:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575749.250683:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575749.250693:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575749.250698:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548382:548382:256:4294967295] 192.168.202.128@tcp LPNI seq info [548305:548305:8:4294967295] 00000400:00000200:0.0:1788575749.250713:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575749.250724:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575749.250735:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575749.250743:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575749.250754:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575749.250758:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575749.250780:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575749.250782:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575749.250785:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575749.250788:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575749.250790:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575749.250792:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575749.250793:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575749.250796:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516861440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.250804:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575749.250806:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516861440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.250828:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.250829:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575749.250836:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575749.250839:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575749.250841:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575749.250843:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311109140 00000100:00000001:0.0:1788575749.250844:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311112941 00000100:00000001:0.0:1788575749.250846:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311109140 00000100:00000001:0.0:1788575749.250848:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311112942 00000100:00000001:0.0:1788575749.250849:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575749.250850:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.250852:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575749.250854:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575749/real 1788575749] req@ffff8fc4ac821880 x1875440516861440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575749.250858:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575749.250859:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575749.250861:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575749.250862:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575749.250863:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575749.250865:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575749.250866:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516861440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575749.250872:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516861440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.250875:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575749.250876:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.250877:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575749.250878:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575749.250881:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516861440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.250887:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575749.250891:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516861440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.250904:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575749.250907:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.250908:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575749.250910:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.250912:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575749.250914:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575749.250916:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575749.250920:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575749.250922:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575749.250926:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516861440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575749.250934:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516861440:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575749.250938:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575749.250942:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516861568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.250949:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575749.250950:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575749.250952:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575749.250954:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575749.250956:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575749.250959:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516861568:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575749.250963:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575749.250965:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575749.250967:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575749.250968:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575749.250972:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575749.250974:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575749.250982:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575749.250985:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575749.250989:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575749.250992:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516861568, portal 10 00000100:00000001:0.0:1788575749.250994:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575749.250995:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788575749.251001:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516861568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251007:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575749.251009:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575749.251011:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575749.251013:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516861568, offset 0 00000400:00000200:0.0:1788575749.251017:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575749.251021:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575749.251025:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575749.251028:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575749.251031:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575749.251033:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575749.251035:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575749.251038:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548383:548383:256:4294967295] 192.168.202.128@tcp LPNI seq info [548306:548306:8:4294967295] 00000400:00000200:0.0:1788575749.251043:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575749.251048:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575749.251053:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575749.251056:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575749.251060:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575749.251063:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575749.251073:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575749.251075:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575749.251078:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575749.251081:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575749.251084:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575749.251090:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575749.251094:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575749.251097:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516861568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251105:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575749.251107:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516861568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251114:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.251115:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575749.251119:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575749.251122:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575749.251125:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575749.251127:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311109147 00000100:00000001:0.0:1788575749.251130:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311112941 00000100:00000001:0.0:1788575749.251132:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311109147 00000100:00000001:0.0:1788575749.251134:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311112942 00000100:00000001:0.0:1788575749.251136:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575749.251138:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.251140:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575749.251143:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575749/real 1788575749] req@ffff8fc4ac820a80 x1875440516861568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575749.251151:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575749.251152:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575749.251154:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575749.251156:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575749.251158:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575749.251160:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575749.251162:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516861568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575749.251169:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516861568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251179:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575749.251181:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251182:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575749.251184:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575749.251187:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516861568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251193:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575749.251196:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516861568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251203:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575749.251205:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251207:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575749.251208:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251210:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575749.251213:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575749.251215:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575749.251218:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575749.251221:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575749.251225:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516861568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575749.251232:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516861568:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575749.251236:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251238:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575749.251241:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516861440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251247:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575749.251254:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575749.251256:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575749.251258:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575749.251260:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575749.251261:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575749.251263:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575749.251264:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788575749.251266:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575749.251267:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575749.251268:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788575749.251270:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575749.251271:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251272:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575749.251274:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516861568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251278:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575749.251279:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575749.251280:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575749.251281:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575749.251282:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575749.251283:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575749.251284:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575749.251285:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788575749.251286:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575749.251286:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575749.251287:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788575749.251288:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575749.251289:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251290:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251292:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575749.251293:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.251294:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575749.251295:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575749.251296:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575749.251298:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516861696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251302:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575749.251306:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575749.251308:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575749.251309:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575749.251310:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575749.251312:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516861696:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575749.251315:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575749.251317:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575749.251318:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575749.251319:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575749.251321:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575749.251322:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575749.251325:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575749.251326:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575749.251328:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575749.251331:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516861696, portal 4 00000100:00000001:0.0:1788575749.251332:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575749.251333:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788575749.251336:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516861696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251340:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575749.251342:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575749.251343:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575749.251344:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516861696, offset 0 00000400:00000200:0.0:1788575749.251347:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575749.251349:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575749.251351:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575749.251353:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575749.251355:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575749.251356:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575749.251358:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575749.251360:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548384:548384:256:4294967295] 192.168.202.128@tcp LPNI seq info [548307:548307:8:4294967295] 00000400:00000200:0.0:1788575749.251362:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575749.251366:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575749.251372:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575749.251374:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575749.251376:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575749.251378:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575749.251385:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575749.251386:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575749.251388:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575749.251390:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575749.251392:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575749.251393:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575749.251395:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575749.251397:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516861696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251401:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575749.251403:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516861696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251407:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.251408:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575749.251409:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575749.251411:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575749.251425:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575749.251427:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311109287 00000100:00000001:0.0:1788575749.251429:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311112941 00000100:00000001:0.0:1788575749.251431:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311109287 00000100:00000001:0.0:1788575749.251432:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311112942 00000100:00000001:0.0:1788575749.251434:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575749.251435:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.251436:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575749.251438:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575749/real 1788575749] req@ffff8fc4ac823800 x1875440516861696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575749.251449:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575749.251450:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575749.251451:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575749.251453:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575749.251454:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575749.251455:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575749.251457:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516861696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575749.251463:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516861696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251466:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575749.251467:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251468:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575749.251469:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575749.251471:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516861696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251475:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575749.251477:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516861696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251481:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575749.251483:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251483:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575749.251484:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251485:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575749.251486:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575749.251487:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575749.251490:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575749.251491:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575749.251496:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516861696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575749.251500:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516861696:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575749.251503:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575749.251505:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516861824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251508:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575749.251509:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575749.251510:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575749.251511:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575749.251512:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575749.251514:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516861824:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575749.251515:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575749.251517:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575749.251518:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575749.251519:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575749.251520:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575749.251522:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575749.251524:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575749.251525:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575749.251527:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575749.251529:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516861824, portal 4 00000100:00000001:0.0:1788575749.251530:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575749.251531:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788575749.251535:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516861824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251539:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575749.251540:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575749.251541:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575749.251543:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516861824, offset 0 00000400:00000200:0.0:1788575749.251548:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575749.251550:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575749.251552:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575749.251554:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575749.251556:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575749.251557:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575749.251559:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575749.251561:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548385:548385:256:4294967295] 192.168.202.128@tcp LPNI seq info [548308:548308:8:4294967295] 00000400:00000200:0.0:1788575749.251563:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575749.251567:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575749.251569:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575749.251571:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575749.251573:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575749.251575:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1788575749.251581:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575749.251582:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575749.251584:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575749.251586:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575749.251588:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575749.251589:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575749.251591:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575749.251593:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516861824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251597:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575749.251599:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516861824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575774 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251603:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.251604:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575749.251606:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575749.251608:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575749.251609:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575749.251613:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311109289 00000100:00000001:0.0:1788575749.251615:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311112941 00000100:00000001:0.0:1788575749.251617:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311109289 00000100:00000001:0.0:1788575749.251618:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311112942 00000100:00000001:0.0:1788575749.251619:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575749.251620:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.251621:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575749.251623:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575749/real 1788575749] req@ffff8fc4ac820e00 x1875440516861824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575749.251628:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575749.251629:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575749.251630:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575749.251631:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575749.251632:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575749.251633:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575749.251635:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516861824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575749.251640:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516861824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251644:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575749.251645:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251645:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575749.251646:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575749.251648:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516861824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251651:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575749.251653:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516861824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251657:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575749.251661:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251662:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575749.251663:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251664:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575749.251665:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575749.251666:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575749.251668:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575749.251669:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575749.251671:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516861824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575749.251676:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516861824:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575749.251678:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251679:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575749.251681:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516861696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251684:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575749.251685:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575749.251686:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575749.251688:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575749.251688:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575749.251689:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575749.251690:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575749.251691:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788575749.251693:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575749.251693:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575749.251694:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788575749.251696:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575749.251697:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251698:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575749.251700:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516861824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575774 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575749.251709:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575749.251710:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575749.251711:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575749.251712:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575749.251713:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575749.251713:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575749.251714:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575749.251715:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788575749.251716:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575749.251717:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575749.251717:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788575749.251719:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575749.251719:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251720:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.251722:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575749.251723:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.251724:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575749.251725:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.760882:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575749.760889:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575749.760893:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575749.760898:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575749.760902:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575749.760905:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.761056:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575749.761060:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.761064:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575749.761157:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575749.761160:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575749.761163:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575749.761168:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575749.761171:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575749.761175:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.761177:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575749.761178:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575749.761180:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575750.785088:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575750.785094:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575750.785097:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575750.785103:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575750.785109:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575750.785126:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575750.785131:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575750.785133:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575750.785139:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575751.808936:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575751.808948:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575751.808959:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575751.808974:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575751.808987:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575751.809000:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575751.809014:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575751.809020:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575751.809023:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575752.513060:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575752.513065:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575752.513069:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575752.832944:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575752.832952:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575752.832960:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575752.832967:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575752.832983:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575752.832995:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575752.833018:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575752.833021:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575752.833025:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575752.833051:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575752.833056:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575752.833058:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575752.833062:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575752.833066:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575752.833071:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575752.833075:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575752.833078:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575752.833110:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575753.088946:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311112941, at 4311112960 00000100:00000010:0.0:1788575753.088966:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575753.537060:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575753.537064:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575753.537067:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575753.856955:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575753.856961:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575753.856964:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575753.856971:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575753.856975:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575753.856980:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575753.856988:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575753.856992:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575753.856996:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.369133:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575754.369139:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575754.369143:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575754.369154:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5180 x1875440516861952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.369200:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575754.369204:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575754.369210:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575754.369214:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575754.369217:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575754.369225:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516861952:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575754.369231:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575754.369237:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575754.369242:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575754.369247:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575754.369253:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575754.369257:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575754.369264:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575754.369270:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575754.369278:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575754.369293:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516861952, portal 25 00000100:00000001:0.0:1788575754.369299:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575754.369301:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605325184 : -123400104226432 : ffff8fc4ac9f5180) 00000100:00000040:0.0:1788575754.369315:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5180 x1875440516861952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.369343:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575754.369349:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575754.369355:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575754.369359:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516861952, offset 0 00000400:00000200:0.0:1788575754.369368:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575754.369379:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575754.369385:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575754.369390:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575754.369394:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575754.369397:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575754.369400:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575754.369403:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548386:548386:256:4294967295] 192.168.202.128@tcp LPNI seq info [548309:548309:8:4294967295] 00000400:00000200:0.0:1788575754.369408:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575754.369416:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575754.369424:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575754.369432:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575754.369443:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575754.369448:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575754.369483:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575754.369486:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575754.369492:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575754.369497:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575754.369503:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575754.369505:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575754.369512:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575754.369515:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5180 x1875440516861952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.369537:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575754.369541:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5180 x1875440516861952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.369554:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.369558:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575754.369567:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575754.369575:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575754.369580:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311118061 00000100:00000010:0.0:1788575754.369588:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575754.369598:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575754.369607:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311118061 00000100:00000010:0.0:1788575754.369615:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50480. 00000100:00000001:0.0:1788575754.369621:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575754.369626:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575754.369631:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.369636:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575754.369645:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575754/real 1788575754] req@ffff8fc4ac9f5180 x1875440516861952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575754.369670:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575754.369672:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575754.369676:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575754.369682:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575754.369688:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575754.369694:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575754.369702:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5180 x1875440516861952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575754.369724:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5180 x1875440516861952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.369738:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575754.369748:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.369753:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575754.369758:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575754.369767:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5180 x1875440516861952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.369782:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575754.369791:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5180 x1875440516861952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.369802:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575754.369830:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.369833:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575754.369835:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.369837:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575754.369840:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575754.369845:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575754.369850:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575754.369855:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575754.369864:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5180 x1875440516861952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575754.369879:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516861952:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575754.369887:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.369892:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575754.369900:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5180 x1875440516861952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.369912:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575754.369915:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575754.369920:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575754.369925:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575754.369928:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575754.369937:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575754.369942:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575754.369947:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35800. 02000000:00000001:0.0:1788575754.369953:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575754.369956:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575754.369959:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5180. 00000100:00000001:0.0:1788575754.369964:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575754.369967:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.369971:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.369979:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575754.369984:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.369990:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575754.369993:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575754.369997:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575754.370004:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5f80 x1875440516862080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370021:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575754.370024:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575754.370028:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575754.370030:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575754.370035:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575754.370040:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862080:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575754.370047:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575754.370050:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575754.370053:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575754.370057:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575754.370062:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575754.370067:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575754.370074:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575754.370080:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575754.370085:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575754.370090:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516862080, portal 10 00000100:00000001:0.0:1788575754.370096:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575754.370100:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605328768 : -123400104222848 : ffff8fc4ac9f5f80) 00000100:00000040:0.0:1788575754.370112:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5f80 x1875440516862080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370133:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575754.370139:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575754.370143:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575754.370148:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516862080, offset 0 00000400:00000200:0.0:1788575754.370158:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575754.370179:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575754.370187:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575754.370193:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575754.370200:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575754.370204:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575754.370206:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575754.370212:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548387:548387:256:4294967295] 192.168.202.128@tcp LPNI seq info [548310:548310:8:4294967295] 00000400:00000200:0.0:1788575754.370223:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575754.370236:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575754.370246:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575754.370252:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575754.370260:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575754.370265:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575754.370280:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575754.370283:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575754.370287:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575754.370291:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575754.370295:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575754.370297:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575754.370300:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575754.370304:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5f80 x1875440516862080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370312:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575754.370315:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5f80 x1875440516862080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370331:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.370333:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575754.370337:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575754.370341:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575754.370345:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575754.370347:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311114259 00000100:00000001:0.0:1788575754.370349:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311118061 00000100:00000001:0.0:1788575754.370352:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311114259 00000100:00000001:0.0:1788575754.370354:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311118061 00000100:00000001:0.0:1788575754.370356:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575754.370358:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.370360:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575754.370364:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575754/real 1788575754] req@ffff8fc4ac9f5f80 x1875440516862080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575754.370372:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575754.370373:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575754.370375:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575754.370377:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575754.370378:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575754.370380:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575754.370383:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5f80 x1875440516862080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575754.370390:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5f80 x1875440516862080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370395:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575754.370397:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.370398:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575754.370400:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575754.370403:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5f80 x1875440516862080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370416:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575754.370420:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5f80 x1875440516862080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370426:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575754.370429:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.370430:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575754.370431:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.370433:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575754.370434:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575754.370437:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575754.370440:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575754.370442:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575754.370446:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5f80 x1875440516862080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575754.370453:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862080:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575754.370457:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575754.370460:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4700 x1875440516862208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370467:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575754.370468:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575754.370470:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575754.370472:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575754.370474:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575754.370477:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862208:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575754.370480:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575754.370482:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575754.370489:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575754.370490:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575754.370493:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575754.370495:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575754.370498:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575754.370501:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575754.370505:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575754.370507:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516862208, portal 10 00000100:00000001:0.0:1788575754.370509:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575754.370510:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605322496 : -123400104229120 : ffff8fc4ac9f4700) 00000100:00000040:0.0:1788575754.370516:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4700 x1875440516862208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370522:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575754.370525:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575754.370526:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575754.370529:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516862208, offset 0 00000400:00000200:0.0:1788575754.370532:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575754.370535:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575754.370538:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575754.370541:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575754.370543:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575754.370545:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575754.370547:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575754.370550:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548388:548388:256:4294967295] 192.168.202.128@tcp LPNI seq info [548311:548311:8:4294967295] 00000400:00000200:0.0:1788575754.370553:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575754.370559:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575754.370563:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575754.370565:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575754.370569:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575754.370572:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575754.370581:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575754.370587:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575754.370590:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575754.370593:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575754.370596:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575754.370597:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575754.370600:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575754.370603:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4700 x1875440516862208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370609:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575754.370612:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4700 x1875440516862208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370618:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.370619:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575754.370622:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575754.370624:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575754.370627:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575754.370629:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311114266 00000100:00000001:0.0:1788575754.370632:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311118061 00000100:00000001:0.0:1788575754.370634:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311114266 00000100:00000001:0.0:1788575754.370636:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311118061 00000100:00000001:0.0:1788575754.370637:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575754.370639:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.370641:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575754.370645:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575754/real 1788575754] req@ffff8fc4ac9f4700 x1875440516862208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575754.370652:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575754.370653:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575754.370656:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575754.370658:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575754.370659:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575754.370665:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575754.370669:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4700 x1875440516862208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575754.370676:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4700 x1875440516862208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370682:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575754.370683:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.370685:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575754.370686:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575754.370690:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4700 x1875440516862208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370697:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575754.370700:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4700 x1875440516862208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370706:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575754.370708:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.370710:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575754.370711:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.370713:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575754.370716:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575754.370718:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575754.370721:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575754.370723:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575754.370726:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4700 x1875440516862208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575754.370733:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862208:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575754.370737:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575754.370744:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7800 x1875440516862336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370749:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575754.370751:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575754.370752:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575754.370754:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575754.370756:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575754.370759:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862336:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575754.370762:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575754.370764:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575754.370765:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575754.370767:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575754.370769:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575754.370771:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575754.370774:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575754.370776:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575754.370779:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575754.370782:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516862336, portal 4 00000100:00000001:0.0:1788575754.370784:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575754.370785:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605335040 : -123400104216576 : ffff8fc4ac9f7800) 00000100:00000040:0.0:1788575754.370790:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7800 x1875440516862336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370796:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575754.370798:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575754.370800:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575754.370802:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516862336, offset 0 00000400:00000200:0.0:1788575754.370805:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575754.370830:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575754.370833:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575754.370836:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575754.370838:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575754.370840:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575754.370842:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575754.370849:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548389:548389:256:4294967295] 192.168.202.128@tcp LPNI seq info [548312:548312:8:4294967295] 00000400:00000200:0.0:1788575754.370853:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575754.370859:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575754.370863:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575754.370867:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575754.370870:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575754.370873:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575754.370888:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575754.370889:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575754.370892:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575754.370895:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575754.370899:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575754.370900:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575754.370903:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575754.370906:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7800 x1875440516862336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370913:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575754.370916:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7800 x1875440516862336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370922:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.370923:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575754.370926:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575754.370929:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575754.370932:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575754.370933:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311114407 00000100:00000001:0.0:1788575754.370936:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311118061 00000100:00000001:0.0:1788575754.370937:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311114407 00000100:00000001:0.0:1788575754.370939:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311118061 00000100:00000001:0.0:1788575754.370941:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575754.370948:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.370950:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575754.370954:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575754/real 1788575754] req@ffff8fc4ac9f7800 x1875440516862336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575754.370961:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575754.370962:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575754.370964:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575754.370966:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575754.370968:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575754.370970:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575754.370973:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7800 x1875440516862336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575754.370980:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7800 x1875440516862336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.370986:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575754.370988:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.370989:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575754.370991:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575754.370994:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7800 x1875440516862336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371000:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575754.371003:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7800 x1875440516862336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371009:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575754.371011:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371012:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575754.371013:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371015:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575754.371017:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575754.371019:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575754.371027:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575754.371029:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575754.371033:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7800 x1875440516862336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575754.371039:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862336:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575754.371043:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371044:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575754.371047:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5f80 x1875440516862080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371053:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575754.371055:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575754.371057:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575754.371060:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575754.371061:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575754.371062:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575754.371064:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575754.371066:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34400. 02000000:00000001:0.0:1788575754.371068:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575754.371070:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575754.371072:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5f80. 00000100:00000001:0.0:1788575754.371074:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575754.371075:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371076:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575754.371079:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4700 x1875440516862208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371085:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575754.371086:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575754.371087:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575754.371089:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575754.371090:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575754.371092:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575754.371093:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575754.371099:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36c00. 02000000:00000001:0.0:1788575754.371102:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575754.371103:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575754.371104:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4700. 00000100:00000001:0.0:1788575754.371106:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575754.371107:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371108:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575754.371112:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7800 x1875440516862336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371118:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575754.371119:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575754.371120:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575754.371123:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575754.371125:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575754.371126:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575754.371128:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575754.371130:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37800. 02000000:00000001:0.0:1788575754.371132:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575754.371132:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575754.371134:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7800. 00000100:00000001:0.0:1788575754.371136:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575754.371137:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371139:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371142:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575754.371144:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.371146:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575754.371147:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575754.371149:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575754.371152:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6a00 x1875440516862464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371158:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575754.371159:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575754.371161:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575754.371162:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575754.371179:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575754.371182:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862464:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575754.371189:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575754.371191:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575754.371193:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575754.371194:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575754.371196:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575754.371198:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575754.371202:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575754.371205:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575754.371208:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575754.371211:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516862464, portal 4 00000100:00000001:0.0:1788575754.371213:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575754.371214:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605331456 : -123400104220160 : ffff8fc4ac9f6a00) 00000100:00000040:0.0:1788575754.371219:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6a00 x1875440516862464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371225:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575754.371227:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575754.371229:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575754.371231:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516862464, offset 0 00000400:00000200:0.0:1788575754.371235:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575754.371239:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575754.371242:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575754.371245:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575754.371247:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575754.371250:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575754.371252:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575754.371255:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548390:548390:256:4294967295] 192.168.202.128@tcp LPNI seq info [548313:548313:8:4294967295] 00000400:00000200:0.0:1788575754.371259:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575754.371265:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575754.371269:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575754.371273:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575754.371276:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575754.371283:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1788575754.371293:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575754.371295:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575754.371298:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575754.371301:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575754.371304:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575754.371306:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575754.371309:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575754.371312:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6a00 x1875440516862464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371319:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575754.371322:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6a00 x1875440516862464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575779 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371328:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.371329:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575754.371332:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575754.371335:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575754.371338:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575754.371340:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311114409 00000100:00000001:0.0:1788575754.371343:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311118061 00000100:00000001:0.0:1788575754.371345:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311114409 00000100:00000001:0.0:1788575754.371347:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311118061 00000100:00000001:0.0:1788575754.371349:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575754.371351:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.371352:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575754.371356:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575754/real 1788575754] req@ffff8fc4ac9f6a00 x1875440516862464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575754.371363:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575754.371364:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575754.371371:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575754.371374:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575754.371375:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575754.371378:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575754.371382:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6a00 x1875440516862464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575754.371389:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6a00 x1875440516862464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371395:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575754.371396:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371398:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575754.371400:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575754.371404:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6a00 x1875440516862464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371410:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575754.371413:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6a00 x1875440516862464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371419:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575754.371421:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371423:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575754.371424:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371426:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575754.371428:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575754.371430:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575754.371433:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575754.371436:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575754.371440:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6a00 x1875440516862464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575754.371446:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862464:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575754.371455:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371456:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575754.371460:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6a00 x1875440516862464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575779 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575754.371465:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575754.371466:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575754.371468:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575754.371470:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575754.371471:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575754.371472:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575754.371474:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575754.371475:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788575754.371478:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575754.371479:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575754.371480:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6a00. 00000100:00000001:0.0:1788575754.371482:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575754.371483:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371485:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.371488:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575754.371490:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.371491:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575754.371493:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575754.880918:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575754.880927:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575754.880934:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575754.880942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575754.880948:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575754.880953:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.880964:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575754.880968:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575754.880972:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575755.905015:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575755.905021:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575755.905024:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575755.905029:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575755.905034:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575755.905062:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575755.905065:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575755.905067:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575755.905241:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575756.928918:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575756.928926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575756.928929:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575756.928935:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575756.928939:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575756.928946:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575756.928952:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575756.928956:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575756.928960:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575756.928994:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575756.928998:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575756.928999:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575756.929001:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575756.929003:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575756.929005:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575756.929007:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575756.929009:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575756.929011:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575757.633020:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575757.633023:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575757.633025:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575757.952912:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575757.952916:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575757.952921:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575757.952928:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575757.952932:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575757.952935:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575757.952953:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575757.952956:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575757.952960:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575758.208899:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311118061, at 4311118080 00000100:00000010:0.0:1788575758.208911:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575758.657039:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575758.657058:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575758.657061:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575758.976953:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575758.976959:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575758.976961:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575758.976965:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575758.976971:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575758.976977:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575758.976982:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575758.976984:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575758.976988:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.489108:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575759.489113:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575759.489118:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575759.489127:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7100 x1875440516862592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.489143:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575759.489147:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575759.489153:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575759.489156:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575759.489161:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575759.489168:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516862592:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575759.489174:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575759.489179:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575759.489182:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575759.489186:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575759.489191:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575759.489194:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575759.489203:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575759.489208:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575759.489215:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575759.489222:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516862592, portal 25 00000100:00000001:0.0:1788575759.489226:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575759.489228:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605333248 : -123400104218368 : ffff8fc4ac9f7100) 00000100:00000040:0.0:1788575759.489239:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7100 x1875440516862592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.489264:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575759.489272:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575759.489276:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575759.489281:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516862592, offset 0 00000400:00000200:0.0:1788575759.489300:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575759.489311:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575759.489318:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575759.489322:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575759.489328:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575759.489334:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575759.489338:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575759.489344:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548391:548391:256:4294967295] 192.168.202.128@tcp LPNI seq info [548314:548314:8:4294967295] 00000400:00000200:0.0:1788575759.489348:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575759.489361:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575759.489371:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575759.489378:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575759.489388:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575759.489393:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1788575759.489418:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575759.489422:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575759.489427:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575759.489434:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575759.489440:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575759.489443:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575759.489448:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575759.489452:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7100 x1875440516862592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.489468:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575759.489477:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7100 x1875440516862592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.489514:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.489520:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575759.489528:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575759.489538:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575759.489545:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311123181 00000100:00000010:0.0:1788575759.489554:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575759.489564:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575759.489573:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311123181 00000100:00000010:0.0:1788575759.489578:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50ea0. 00000100:00000001:0.0:1788575759.489582:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575759.489585:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575759.489589:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.489593:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575759.489599:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575759/real 1788575759] req@ffff8fc4ac9f7100 x1875440516862592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575759.489610:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575759.489612:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575759.489615:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575759.489617:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575759.489620:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575759.489622:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575759.489628:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7100 x1875440516862592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575759.489644:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7100 x1875440516862592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.489657:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575759.489660:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.489664:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575759.489668:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575759.489677:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7100 x1875440516862592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.489701:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575759.489712:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7100 x1875440516862592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.489731:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575759.489739:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.489744:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575759.489747:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.489752:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575759.489757:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575759.489764:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575759.489770:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575759.489776:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575759.489785:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7100 x1875440516862592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575759.489805:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516862592:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575759.489832:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.489835:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575759.489841:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7100 x1875440516862592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.489857:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575759.489861:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575759.489864:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575759.489869:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575759.489873:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575759.489876:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575759.489881:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575759.489884:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34000. 02000000:00000001:0.0:1788575759.489892:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575759.489900:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575759.489904:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7100. 00000100:00000001:0.0:1788575759.489910:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575759.489925:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.489931:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.489939:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575759.489943:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.489946:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575759.489948:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575759.489950:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575759.489955:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f4000 x1875440516862720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.489963:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575759.489964:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575759.489966:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575759.489968:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575759.489970:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575759.489973:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575759.489976:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575759.489978:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575759.489980:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575759.489982:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575759.489985:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575759.489987:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575759.489991:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575759.489994:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575759.489997:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575759.490000:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516862720, portal 10 00000100:00000001:0.0:1788575759.490002:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575759.490004:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605320704 : -123400104230912 : ffff8fc4ac9f4000) 00000100:00000040:0.0:1788575759.490010:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f4000 x1875440516862720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490017:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575759.490019:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575759.490026:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575759.490028:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516862720, offset 0 00000400:00000200:0.0:1788575759.490032:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575759.490036:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575759.490040:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575759.490043:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575759.490046:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575759.490048:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575759.490050:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575759.490052:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548392:548392:256:4294967295] 192.168.202.128@tcp LPNI seq info [548315:548315:8:4294967295] 00000400:00000200:0.0:1788575759.490055:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575759.490059:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575759.490062:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575759.490064:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575759.490067:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575759.490069:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1788575759.490076:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575759.490077:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575759.490079:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575759.490081:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575759.490083:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575759.490084:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575759.490086:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575759.490089:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f4000 x1875440516862720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490094:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575759.490096:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f4000 x1875440516862720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490100:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.490101:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575759.490108:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575759.490111:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575759.490113:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575759.490115:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311119379 00000100:00000001:0.0:1788575759.490116:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311123181 00000100:00000001:0.0:1788575759.490118:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311119379 00000100:00000001:0.0:1788575759.490119:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311123181 00000100:00000001:0.0:1788575759.490120:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575759.490121:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.490123:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575759.490125:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575759/real 1788575759] req@ffff8fc4ac9f4000 x1875440516862720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575759.490130:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575759.490130:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575759.490132:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575759.490133:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575759.490134:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575759.490136:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575759.490137:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f4000 x1875440516862720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575759.490142:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f4000 x1875440516862720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490146:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575759.490146:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490147:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575759.490148:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575759.490150:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f4000 x1875440516862720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490153:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575759.490156:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f4000 x1875440516862720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490163:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575759.490164:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490165:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575759.490166:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490167:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575759.490168:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575759.490170:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575759.490172:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575759.490173:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575759.490175:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f4000 x1875440516862720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575759.490180:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862720:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575759.490182:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575759.490184:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f6680 x1875440516862848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490187:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575759.490188:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575759.490189:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575759.490191:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575759.490192:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575759.490193:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862848:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575759.490195:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575759.490196:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575759.490197:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575759.490198:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575759.490199:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575759.490200:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575759.490205:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575759.490206:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575759.490208:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575759.490210:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516862848, portal 10 00000100:00000001:0.0:1788575759.490212:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575759.490212:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605330560 : -123400104221056 : ffff8fc4ac9f6680) 00000100:00000040:0.0:1788575759.490217:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f6680 x1875440516862848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490221:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575759.490222:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575759.490223:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575759.490225:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516862848, offset 0 00000400:00000200:0.0:1788575759.490227:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575759.490230:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575759.490233:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575759.490235:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575759.490238:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575759.490240:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575759.490242:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575759.490245:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548393:548393:256:4294967295] 192.168.202.128@tcp LPNI seq info [548316:548316:8:4294967295] 00000400:00000200:0.0:1788575759.490249:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575759.490254:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575759.490259:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575759.490262:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575759.490266:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575759.490268:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1788575759.490279:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575759.490281:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575759.490283:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575759.490285:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575759.490288:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575759.490292:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575759.490294:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575759.490297:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f6680 x1875440516862848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490302:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575759.490304:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f6680 x1875440516862848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490308:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.490309:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575759.490311:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575759.490313:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575759.490315:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575759.490316:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311119386 00000100:00000001:0.0:1788575759.490318:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311123181 00000100:00000001:0.0:1788575759.490319:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311119386 00000100:00000001:0.0:1788575759.490321:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311123181 00000100:00000001:0.0:1788575759.490322:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575759.490323:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.490324:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575759.490326:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575759/real 1788575759] req@ffff8fc4ac9f6680 x1875440516862848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575759.490331:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575759.490331:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575759.490333:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575759.490334:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575759.490335:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575759.490336:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575759.490338:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f6680 x1875440516862848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575759.490343:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f6680 x1875440516862848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490350:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575759.490351:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490352:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575759.490353:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575759.490355:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f6680 x1875440516862848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490359:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575759.490361:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f6680 x1875440516862848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490365:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575759.490366:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490367:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575759.490368:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490369:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575759.490370:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575759.490371:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575759.490373:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575759.490374:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575759.490376:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f6680 x1875440516862848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575759.490380:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862848:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575759.490383:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575759.490384:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f7480 x1875440516862976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490388:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575759.490389:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575759.490393:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575759.490394:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575759.490395:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575759.490397:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575759.490399:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575759.490401:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575759.490402:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575759.490403:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575759.490405:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575759.490406:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575759.490408:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575759.490410:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575759.490412:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575759.490414:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516862976, portal 4 00000100:00000001:0.0:1788575759.490415:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575759.490416:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605334144 : -123400104217472 : ffff8fc4ac9f7480) 00000100:00000040:0.0:1788575759.490419:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f7480 x1875440516862976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490423:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575759.490425:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575759.490426:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575759.490427:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516862976, offset 0 00000400:00000200:0.0:1788575759.490429:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575759.490431:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575759.490433:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575759.490435:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575759.490437:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575759.490438:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575759.490440:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575759.490442:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548394:548394:256:4294967295] 192.168.202.128@tcp LPNI seq info [548317:548317:8:4294967295] 00000400:00000200:0.0:1788575759.490444:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575759.490448:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575759.490454:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575759.490456:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575759.490458:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575759.490460:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1788575759.490466:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575759.490468:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575759.490470:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575759.490472:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575759.490473:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575759.490474:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575759.490476:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575759.490478:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f7480 x1875440516862976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490483:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575759.490484:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f7480 x1875440516862976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490488:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.490489:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575759.490491:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575759.490493:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575759.490494:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575759.490495:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311119526 00000100:00000001:0.0:1788575759.490497:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311123181 00000100:00000001:0.0:1788575759.490499:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311119526 00000100:00000001:0.0:1788575759.490500:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311123181 00000100:00000001:0.0:1788575759.490501:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575759.490502:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.490503:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575759.490505:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575759/real 1788575759] req@ffff8fc4ac9f7480 x1875440516862976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575759.490514:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575759.490515:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575759.490516:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575759.490517:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575759.490518:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575759.490520:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575759.490522:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f7480 x1875440516862976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575759.490526:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f7480 x1875440516862976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490529:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575759.490530:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490531:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575759.490532:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575759.490534:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f7480 x1875440516862976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490538:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575759.490540:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f7480 x1875440516862976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490543:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575759.490545:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490545:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575759.490546:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490547:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575759.490548:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575759.490549:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575759.490551:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575759.490552:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575759.490557:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f7480 x1875440516862976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575759.490562:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516862976:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575759.490564:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575759.490566:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac9f5c00 x1875440516863104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490569:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575759.490570:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575759.490571:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575759.490572:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575759.490573:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575759.490575:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516863104:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575759.490576:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575759.490577:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575759.490578:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575759.490579:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575759.490581:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575759.490582:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575759.490584:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575759.490585:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575759.490587:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575759.490589:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516863104, portal 4 00000100:00000001:0.0:1788575759.490590:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575759.490591:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673605327872 : -123400104223744 : ffff8fc4ac9f5c00) 00000100:00000040:0.0:1788575759.490595:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac9f5c00 x1875440516863104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490599:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575759.490600:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575759.490601:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575759.490602:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516863104, offset 0 00000400:00000200:0.0:1788575759.490607:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575759.490609:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575759.490611:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575759.490613:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575759.490615:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575759.490616:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575759.490618:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575759.490619:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548395:548395:256:4294967295] 192.168.202.128@tcp LPNI seq info [548318:548318:8:4294967295] 00000400:00000200:0.0:1788575759.490622:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575759.490626:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575759.490628:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575759.490630:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575759.490632:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575759.490634:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1788575759.490640:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575759.490642:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575759.490643:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575759.490645:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575759.490647:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575759.490648:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575759.490650:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575759.490652:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac9f5c00 x1875440516863104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490656:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575759.490658:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac9f5c00 x1875440516863104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575784 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490662:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.490663:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575759.490664:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575759.490666:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575759.490668:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575759.490672:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311119528 00000100:00000001:0.0:1788575759.490673:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311123181 00000100:00000001:0.0:1788575759.490675:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311119528 00000100:00000001:0.0:1788575759.490676:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311123181 00000100:00000001:0.0:1788575759.490677:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575759.490678:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.490679:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575759.490682:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575759/real 1788575759] req@ffff8fc4ac9f5c00 x1875440516863104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575759.490686:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575759.490687:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575759.490688:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575759.490689:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575759.490690:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575759.490692:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575759.490694:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac9f5c00 x1875440516863104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575759.490699:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac9f5c00 x1875440516863104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490702:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575759.490702:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490703:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575759.490704:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575759.490706:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac9f5c00 x1875440516863104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490709:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575759.490711:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac9f5c00 x1875440516863104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490715:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575759.490718:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490720:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575759.490720:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490721:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575759.490723:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575759.490725:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575759.490726:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575759.490727:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575759.490729:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac9f5c00 x1875440516863104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575759.490734:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac9f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516863104:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575759.490736:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490737:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575759.490739:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f4000 x1875440516862720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490743:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575759.490744:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575759.490745:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575759.490746:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575759.490747:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575759.490748:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575759.490749:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575759.490750:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c35400. 02000000:00000001:0.0:1788575759.490752:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575759.490753:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575759.490754:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f4000. 00000100:00000001:0.0:1788575759.490755:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575759.490756:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490757:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575759.490759:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f6680 x1875440516862848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490771:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575759.490773:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575759.490774:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575759.490776:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575759.490778:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575759.490779:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575759.490781:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575759.490782:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c37c00. 02000000:00000001:0.0:1788575759.490784:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575759.490785:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575759.490787:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f6680. 00000100:00000001:0.0:1788575759.490789:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575759.490790:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490792:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575759.490796:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f7480 x1875440516862976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490802:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575759.490804:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575759.490805:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575759.490826:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575759.490828:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575759.490829:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575759.490830:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575759.490831:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c34800. 02000000:00000001:0.0:1788575759.490833:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575759.490833:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575759.490834:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f7480. 00000100:00000001:0.0:1788575759.490836:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575759.490836:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490838:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575759.490841:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac9f5c00 x1875440516863104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575784 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575759.490845:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575759.490845:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575759.490850:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575759.490851:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575759.490852:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575759.490853:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575759.490855:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575759.490856:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc482c36800. 02000000:00000001:0.0:1788575759.490857:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575759.490858:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575759.490858:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac9f5c00. 00000100:00000001:0.0:1788575759.490859:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575759.490860:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490861:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575759.490864:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575759.490865:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575759.490866:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575759.490867:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575760.000900:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575760.000907:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575760.000910:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575760.000914:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575760.000920:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575760.000925:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575760.000939:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575760.000943:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575760.000947:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575760.000987:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575760.000991:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575760.000992:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575760.000995:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575760.000997:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575760.000999:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575760.001001:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575760.001003:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575760.001005:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575761.024951:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575761.024958:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575761.024962:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575761.024974:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575761.025009:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575761.025016:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575761.025024:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575761.025030:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575761.025033:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575761.536977:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788575761.536990:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788575761.536996:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788575761.537213:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575761.537232:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575761.537238:0:2391:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575761.537241:0:2391:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575762.048918:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575762.048925:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575762.048930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575762.048936:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575762.048943:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575762.048947:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575762.048958:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575762.048962:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575762.048964:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575762.753040:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575762.753044:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575762.753048:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575763.072969:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575763.072974:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575763.072978:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575763.072985:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575763.072991:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575763.072994:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575763.073013:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575763.073017:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575763.073019:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575763.328934:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311123181, at 4311123200 00000100:00000010:0.0:1788575763.328956:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575763.777070:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575763.777079:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575763.777082:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575764.096986:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575764.096990:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575764.096992:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575764.096998:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575764.097001:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575764.097003:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575764.097006:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575764.097010:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575764.097013:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575764.608945:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575764.608955:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575764.608959:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575764.608962:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575764.608968:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575764.608975:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16452 00000400:00000200:0.0:1788575764.608985:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575764.608993:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575764.608996:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575764.609000:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609004:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575764.609007:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609010:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575764.609015:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575764.609017:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575764.609020:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575764.609024:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575764.609026:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575764.609029:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575764.609032:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609034:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575764.609059:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609061:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575764.609065:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575764.609069:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609071:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575764.609074:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575764.609078:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575764.609081:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.609083:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575764.609086:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575764.609095:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575764.609099:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575764.609103:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575764.609105:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.609108:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575764.609111:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575764.609116:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575764.609126:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575764.609135:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575764.609146:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516863232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575764.609158:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575764.609163:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575764.609168:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516863232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575764.609191:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575764.609195:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575764.609198:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575764.609199:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575764.609201:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575764.609203:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575764.609212:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16452 00000400:00000200:0.0:1788575764.609216:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575764.609219:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575764.609220:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575764.609224:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609227:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575764.609230:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609234:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575764.609236:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575764.609239:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575764.609249:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575764.609252:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575764.609256:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575764.609259:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575764.609265:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609268:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575764.609272:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609278:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575764.609283:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575764.609289:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609294:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575764.609299:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575764.609306:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575764.609312:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.609315:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575764.609320:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575764.609325:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.609330:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575764.609336:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575764.609342:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575764.609345:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575764.609352:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.609356:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575764.609358:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575764.609361:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575764.609369:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575764.609376:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575764.609388:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516863360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575764.609400:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575764.609402:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575764.609409:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516863360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575764.609420:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575764.609423:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575764.609425:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575764.609425:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575764.609427:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575764.609430:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575764.609435:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16452 00000400:00000200:0.0:1788575764.609441:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575764.609448:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575764.609450:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575764.609454:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609456:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575764.609461:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609463:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575764.609466:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575764.609469:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575764.609472:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575764.609476:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575764.609480:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575764.609488:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575764.609492:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609496:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575764.609501:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609505:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575764.609509:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575764.609514:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609516:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575764.609517:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575764.609521:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575764.609524:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.609526:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575764.609528:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575764.609534:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.609540:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575764.609544:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575764.609547:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575764.609549:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575764.609550:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.609553:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575764.609554:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575764.609570:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575764.609581:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575764.609589:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575764.609599:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516863488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575764.609616:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575764.609621:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575764.609629:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516863488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575764.609635:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575764.609646:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575764.609652:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575764.609656:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575764.609659:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575764.609676:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575764.609683:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16452 00000400:00000200:0.0:1788575764.609693:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575764.609699:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575764.609704:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575764.609708:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609711:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575764.609716:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609720:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575764.609723:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575764.609725:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575764.609728:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575764.609731:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575764.609734:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575764.609737:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575764.609741:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609743:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575764.609745:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609748:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575764.609753:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575764.609757:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575764.609760:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575764.609762:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575764.609766:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575764.609770:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.609774:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575764.609785:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575764.609788:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.609791:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575764.609796:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575764.609801:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575764.609873:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575764.609875:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.609879:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575764.609880:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575764.609883:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575764.609889:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575764.609895:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575764.609903:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516863616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575764.609920:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575764.609924:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575764.609932:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516863616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575764.609952:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575764.609957:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575764.609965:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575764.609970:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575764.609973:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575764.609980:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575764.609986:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16452 00000400:00000200:0.0:1788575764.609997:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575764.610005:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575764.610010:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575764.610018:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.610025:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575764.610030:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.610042:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575764.610047:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575764.610050:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575764.610054:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575764.610059:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575764.610064:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575764.610069:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575764.610076:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.610083:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575764.610087:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575764.610094:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575764.610100:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575764.610108:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575764.610115:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575764.610121:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575764.610126:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575764.610132:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.610137:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575764.610141:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575764.610148:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.610151:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575764.610157:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575764.610162:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575764.610166:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575764.610169:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575764.610172:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575764.610176:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575764.610180:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575764.610189:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575764.610196:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575764.610206:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516863744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575764.610246:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575764.610251:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575764.610260:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516863744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575764.610277:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575764.610286:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16462) 00000100:00000001:0.0:1788575765.120904:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575765.120913:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575765.120916:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575765.120924:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575765.120931:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575765.120937:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575765.120949:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575765.120952:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575765.120956:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575765.120972:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575765.120977:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575765.120980:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575765.120984:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575765.120987:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575765.120989:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575765.120990:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575765.120992:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575765.120994:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575766.144916:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575766.144925:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575766.144930:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575766.144937:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575766.144943:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575766.144948:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575766.144967:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575766.144970:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575766.144973:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575767.168967:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575767.168972:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575767.168989:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575767.168993:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575767.168995:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575767.168998:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575767.169002:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575767.169006:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575767.169012:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575767.873019:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575767.873022:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575767.873025:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575768.192929:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575768.192938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575768.192942:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575768.192949:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575768.192953:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575768.192963:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575768.192981:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575768.192985:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575768.192987:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575768.448930:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311128301, at 4311128320 00000100:00000010:0.0:1788575768.448950:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfcc0. 00000020:00000001:0.0:1788575768.897036:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575768.897040:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575768.897043:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575769.216932:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575769.216938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575769.216941:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575769.216945:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575769.216950:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575769.216954:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575769.216958:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575769.216962:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575769.216965:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575769.728926:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575769.728940:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575769.728943:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575769.728965:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575769.728971:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575769.728979:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16457 00000400:00000200:0.0:1788575769.728988:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575769.728993:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575769.728996:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575769.728998:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729001:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575769.729002:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729004:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575769.729008:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575769.729010:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575769.729012:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575769.729014:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575769.729016:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575769.729018:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575769.729019:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729021:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575769.729022:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729023:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575769.729026:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575769.729029:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729030:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575769.729033:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575769.729035:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575769.729037:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729039:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575769.729041:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575769.729046:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575769.729049:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575769.729053:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575769.729058:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729070:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575769.729074:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575769.729079:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575769.729089:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575769.729097:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575769.729110:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516863872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575769.729122:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575769.729135:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575769.729142:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516863872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575769.729168:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575769.729172:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575769.729176:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575769.729177:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575769.729178:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575769.729181:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575769.729183:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16457 00000400:00000200:0.0:1788575769.729188:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575769.729191:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575769.729193:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575769.729200:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729202:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575769.729204:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729206:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575769.729208:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575769.729209:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575769.729210:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575769.729213:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575769.729215:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575769.729222:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575769.729225:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729227:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575769.729228:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729230:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575769.729232:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575769.729234:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729236:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575769.729238:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575769.729245:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575769.729253:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729255:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575769.729260:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575769.729263:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729265:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575769.729268:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575769.729270:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575769.729273:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575769.729274:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729275:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575769.729277:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575769.729279:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575769.729282:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575769.729286:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575769.729291:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516864000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575769.729298:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575769.729300:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575769.729304:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516864000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575769.729312:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575769.729325:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575769.729329:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575769.729330:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575769.729332:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575769.729335:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575769.729338:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16457 00000400:00000200:0.0:1788575769.729342:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575769.729345:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575769.729347:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575769.729350:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729353:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575769.729355:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729357:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575769.729359:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575769.729360:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575769.729362:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575769.729364:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575769.729365:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575769.729368:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575769.729371:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729373:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575769.729374:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729376:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575769.729378:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575769.729380:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729382:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575769.729384:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575769.729387:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575769.729390:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729392:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575769.729400:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575769.729402:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729403:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575769.729407:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575769.729409:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575769.729412:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575769.729423:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729425:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575769.729426:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575769.729428:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575769.729433:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575769.729436:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575769.729441:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516864128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575769.729449:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575769.729451:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575769.729463:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516864128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575769.729471:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575769.729473:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575769.729477:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575769.729479:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575769.729480:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575769.729483:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575769.729486:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16457 00000400:00000200:0.0:1788575769.729490:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575769.729493:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575769.729495:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575769.729498:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729500:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575769.729501:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729508:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575769.729511:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575769.729512:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575769.729513:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575769.729515:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575769.729517:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575769.729519:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575769.729522:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729524:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575769.729526:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729527:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575769.729529:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575769.729532:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729534:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575769.729536:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575769.729539:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575769.729541:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729543:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575769.729546:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575769.729548:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729549:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575769.729553:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575769.729555:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575769.729557:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575769.729558:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729559:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575769.729561:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575769.729575:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575769.729580:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575769.729584:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575769.729589:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516864256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575769.729607:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575769.729609:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575769.729613:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516864256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575769.729620:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575769.729622:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575769.729626:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575769.729627:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575769.729629:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575769.729631:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575769.729633:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16457 00000400:00000200:0.0:1788575769.729637:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575769.729641:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575769.729643:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575769.729645:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729648:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575769.729650:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729652:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575769.729653:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575769.729655:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575769.729656:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575769.729658:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575769.729660:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575769.729662:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575769.729665:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729667:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575769.729669:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729671:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575769.729678:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575769.729681:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575769.729683:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575769.729685:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575769.729688:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575769.729690:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729693:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575769.729695:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575769.729697:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729698:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575769.729702:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575769.729704:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575769.729706:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575769.729707:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575769.729709:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575769.729711:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575769.729713:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575769.729717:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575769.729720:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575769.729725:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516864384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575769.729731:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575769.729733:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575769.729737:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516864384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575769.729743:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575769.729749:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16467) 00000100:00000001:0.0:1788575770.240928:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575770.240938:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575770.240942:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575770.240950:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575770.240957:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575770.240979:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575770.240996:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575770.241000:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575770.241003:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575771.265008:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575771.265013:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575771.265017:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575771.265023:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575771.265028:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575771.265032:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575771.265039:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575771.265043:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575771.265058:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575772.289157:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575772.289159:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575772.289160:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575772.289162:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575772.289163:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575772.289165:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575772.289166:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575772.289167:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575772.289168:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575772.993057:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575772.993062:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575772.993065:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575773.312937:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575773.312946:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575773.312949:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575773.312956:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575773.312960:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575773.312964:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575773.312969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575773.312972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575773.312975:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575773.313085:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575773.313088:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575773.313089:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575773.313095:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575773.313131:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575773.313133:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575773.313134:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575773.313136:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575773.313138:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575773.568922:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311133422, at 4311133440 00000100:00000010:0.0:1788575773.568939:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf720. 00000020:00000001:0.0:1788575774.016989:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575774.016992:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575774.016995:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575775.360962:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575775.360968:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575775.360974:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575775.360990:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575775.360997:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575775.361001:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575775.361017:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575775.361021:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575775.361026:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575777.408959:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575777.408963:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575777.408966:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575777.408972:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575777.408980:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575777.408986:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575777.409014:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575777.409017:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575777.409023:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575778.112981:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575778.112988:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575778.112993:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575778.432948:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575778.432954:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575778.432962:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575778.432973:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575778.433001:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575778.433007:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575778.433027:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575778.433031:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575778.433037:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575778.688910:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311138541, at 4311138560 00000100:00000010:0.0:1788575778.688928:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfea0. 00000020:00000001:0.0:1788575779.136976:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575779.136979:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575779.136981:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575779.456918:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575779.456925:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575779.456931:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575779.456967:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575779.456976:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575779.456986:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575779.457010:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575779.457013:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575779.457018:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575780.480997:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575780.481007:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575780.481012:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575780.481021:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575780.481029:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575780.481037:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575780.481042:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575780.481046:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575780.481051:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575781.504972:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575781.504982:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575781.504995:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575781.505011:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575781.505028:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575781.505037:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575781.505067:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575781.505071:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575781.505075:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575782.528983:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575782.528988:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575782.529006:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575782.529012:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575782.529016:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575782.529018:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575782.529020:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575782.529024:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575782.529028:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575782.529058:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575782.529062:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575782.529064:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575782.529066:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575782.529069:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575782.529073:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575782.529087:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575782.529090:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575782.529095:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575783.233040:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575783.233042:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575783.233045:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575783.552970:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575783.552974:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575783.552978:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575783.552984:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575783.552988:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575783.552993:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575783.552997:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575783.552998:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575783.553001:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575783.808889:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311143662, at 4311143680 00000100:00000010:0.0:1788575783.808905:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf420. 00000020:00000001:0.0:1788575784.256963:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575784.256966:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575784.256969:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575784.576913:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575784.576921:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575784.576926:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575784.576934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575784.576961:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575784.576963:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575784.576965:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575784.576966:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575784.576967:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575785.600893:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575785.600900:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575785.600902:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575785.600908:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575785.600913:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575785.600917:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575785.600921:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575785.600924:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575785.600928:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575786.305023:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575786.305026:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575786.305028:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575786.624975:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575786.624979:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575786.624980:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575786.624983:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575786.624985:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575786.624987:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575786.624989:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575786.624991:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575786.624993:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575787.648970:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575787.648979:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575787.648984:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575787.648990:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575787.648994:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575787.648997:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575787.649000:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575787.649010:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575787.649018:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575787.649235:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575787.649237:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575787.649238:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575787.649254:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575787.649256:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575787.649258:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575787.649260:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575787.649261:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575787.649263:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575788.353086:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575788.353093:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575788.353097:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575788.672915:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575788.672921:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575788.672925:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575788.672933:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575788.672944:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575788.672952:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575788.672977:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575788.672980:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575788.672985:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575788.673018:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575788.673023:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575788.673025:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575788.673028:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575788.673031:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575788.673034:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575788.673037:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575788.673039:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575788.673042:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575788.928945:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311148781, at 4311148800 00000100:00000010:0.0:1788575788.928962:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cff00. 00000020:00000001:0.0:1788575789.377052:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575789.377056:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575789.377060:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575789.696919:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575789.696924:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575789.696928:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575789.696933:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575789.696937:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575789.696953:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575789.696957:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575789.696961:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575789.696965:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.209072:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575790.209078:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575790.209082:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575790.209093:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516866432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.209106:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575790.209108:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575790.209114:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575790.209119:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575790.209124:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575790.209132:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516866432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575790.209142:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575790.209148:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575790.209152:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575790.209159:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575790.209165:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575790.209171:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575790.209180:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575790.209188:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575790.209205:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575790.209215:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516866432, portal 25 00000100:00000001:0.0:1788575790.209220:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575790.209224:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575790.209231:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516866432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.209249:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575790.209255:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575790.209259:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575790.209265:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516866432, offset 0 00000400:00000200:0.0:1788575790.209275:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575790.209301:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575790.209312:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575790.209322:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575790.209329:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575790.209337:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575790.209343:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575790.209351:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548421:548421:256:4294967295] 192.168.202.128@tcp LPNI seq info [548344:548344:8:4294967295] 00000400:00000200:0.0:1788575790.209371:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575790.209386:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575790.209394:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575790.209399:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575790.209409:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575790.209414:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575790.209429:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575790.209433:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575790.209437:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575790.209443:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575790.209449:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575790.209452:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575790.209458:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575790.209464:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516866432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.209491:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575790.209496:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516866432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.209507:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.209509:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575790.209515:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575790.209520:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575790.209524:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311153901 00000100:00000010:0.0:1788575790.209529:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575790.209541:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575790.209547:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311153901 00000100:00000010:0.0:1788575790.209552:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50ba0. 00000100:00000001:0.0:1788575790.209555:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575790.209559:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575790.209564:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.209568:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575790.209575:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575790/real 1788575790] req@ffff8fc4901a5880 x1875440516866432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575790.209589:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575790.209591:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575790.209594:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575790.209598:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575790.209602:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575790.209605:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575790.209611:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516866432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575790.209639:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516866432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.209649:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575790.209652:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.209655:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575790.209658:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575790.209665:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516866432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.209676:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575790.209682:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516866432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.209695:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575790.209699:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.209707:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575790.209709:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.209713:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575790.209716:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575790.209718:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575790.209724:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575790.209726:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575790.209732:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516866432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575790.209753:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516866432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575790.209759:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.209763:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575790.209767:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516866432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.209771:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575790.209773:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575790.209776:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575790.209780:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575790.209783:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575790.209786:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575790.209788:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575790.209791:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575790.209795:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575790.209797:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575790.209799:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575790.209805:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575790.209824:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.209825:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.209832:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575790.209834:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.209839:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575790.209840:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575790.209843:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575790.209853:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516866560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.209865:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575790.209868:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575790.209872:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575790.209875:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575790.209878:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575790.209883:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516866560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575790.209891:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575790.209896:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575790.209900:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575790.209906:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575790.209912:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575790.209917:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575790.209926:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575790.209932:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575790.209938:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575790.209944:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516866560, portal 10 00000100:00000001:0.0:1788575790.209948:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575790.209950:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575790.209960:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516866560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.209976:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575790.209980:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575790.209986:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575790.209989:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516866560, offset 0 00000400:00000200:0.0:1788575790.209994:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575790.210000:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575790.210005:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575790.210009:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575790.210013:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575790.210016:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575790.210023:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575790.210026:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548422:548422:256:4294967295] 192.168.202.128@tcp LPNI seq info [548345:548345:8:4294967295] 00000400:00000200:0.0:1788575790.210029:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575790.210038:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575790.210042:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575790.210045:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575790.210049:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575790.210051:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575790.210061:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575790.210063:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575790.210066:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575790.210070:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575790.210072:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575790.210074:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575790.210076:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575790.210079:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516866560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210085:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575790.210088:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516866560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210093:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.210095:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575790.210097:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575790.210099:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575790.210101:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575790.210103:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311150099 00000100:00000001:0.0:1788575790.210104:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311153901 00000100:00000001:0.0:1788575790.210106:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311150099 00000100:00000001:0.0:1788575790.210107:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311153901 00000100:00000001:0.0:1788575790.210112:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575790.210114:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.210115:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575790.210119:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575790/real 1788575790] req@ffff8fc4901a4700 x1875440516866560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575790.210127:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575790.210128:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575790.210130:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575790.210132:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575790.210134:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575790.210136:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575790.210139:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516866560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575790.210148:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516866560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210155:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575790.210156:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210158:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575790.210159:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575790.210163:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516866560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210168:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575790.210170:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516866560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210174:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575790.210176:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210177:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575790.210177:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210178:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575790.210180:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575790.210185:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575790.210188:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575790.210189:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575790.210192:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516866560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575790.210197:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516866560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575790.210199:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575790.210201:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516866688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210204:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575790.210205:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575790.210207:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575790.210208:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575790.210209:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575790.210211:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516866688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575790.210213:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575790.210214:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575790.210215:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575790.210217:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575790.210219:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575790.210220:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575790.210222:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575790.210224:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575790.210226:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575790.210228:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516866688, portal 10 00000100:00000001:0.0:1788575790.210229:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575790.210230:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575790.210234:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516866688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210244:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575790.210246:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575790.210247:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575790.210249:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516866688, offset 0 00000400:00000200:0.0:1788575790.210251:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575790.210253:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575790.210256:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575790.210258:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575790.210259:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575790.210261:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575790.210262:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575790.210264:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548423:548423:256:4294967295] 192.168.202.128@tcp LPNI seq info [548346:548346:8:4294967295] 00000400:00000200:0.0:1788575790.210267:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575790.210270:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575790.210273:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575790.210274:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575790.210277:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575790.210278:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575790.210285:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575790.210286:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575790.210288:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575790.210290:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575790.210292:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575790.210293:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575790.210295:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575790.210297:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516866688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210303:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575790.210305:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516866688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210321:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.210323:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575790.210327:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575790.210330:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575790.210339:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575790.210341:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311150106 00000100:00000001:0.0:1788575790.210344:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311153901 00000100:00000001:0.0:1788575790.210346:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311150106 00000100:00000001:0.0:1788575790.210348:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311153901 00000100:00000001:0.0:1788575790.210350:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575790.210351:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.210353:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575790.210356:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575790/real 1788575790] req@ffff8fc4901a7480 x1875440516866688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575790.210361:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575790.210362:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575790.210363:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575790.210365:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575790.210366:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575790.210367:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575790.210369:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516866688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575790.210374:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516866688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210378:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575790.210378:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210379:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575790.210380:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575790.210382:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516866688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210391:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575790.210393:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516866688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210397:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575790.210399:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210400:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575790.210401:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210402:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575790.210404:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575790.210405:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575790.210407:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575790.210408:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575790.210410:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516866688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575790.210414:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516866688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575790.210417:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575790.210419:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516866816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210422:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575790.210423:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575790.210424:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575790.210426:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575790.210427:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575790.210428:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516866816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575790.210430:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575790.210431:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575790.210435:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575790.210437:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575790.210439:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575790.210440:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575790.210442:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575790.210444:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575790.210446:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575790.210448:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516866816, portal 4 00000100:00000001:0.0:1788575790.210449:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575790.210450:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575790.210454:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516866816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210458:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575790.210459:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575790.210460:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575790.210462:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516866816, offset 0 00000400:00000200:0.0:1788575790.210464:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575790.210466:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575790.210468:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575790.210470:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575790.210471:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575790.210473:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575790.210475:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575790.210477:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548424:548424:256:4294967295] 192.168.202.128@tcp LPNI seq info [548347:548347:8:4294967295] 00000400:00000200:0.0:1788575790.210479:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575790.210483:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575790.210486:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575790.210488:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575790.210490:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575790.210492:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575790.210499:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575790.210503:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575790.210504:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575790.210507:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575790.210509:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575790.210510:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575790.210511:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575790.210514:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516866816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210518:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575790.210520:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516866816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210524:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.210525:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575790.210528:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575790.210530:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575790.210533:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575790.210535:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311150246 00000100:00000001:0.0:1788575790.210537:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311153901 00000100:00000001:0.0:1788575790.210539:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311150246 00000100:00000001:0.0:1788575790.210541:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311153901 00000100:00000001:0.0:1788575790.210543:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575790.210545:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.210547:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575790.210551:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575790/real 1788575790] req@ffff8fc4901a6300 x1875440516866816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575790.210558:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575790.210560:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575790.210562:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575790.210564:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575790.210565:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575790.210574:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575790.210578:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516866816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575790.210586:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516866816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210592:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575790.210593:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210596:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575790.210597:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575790.210600:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516866816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210607:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575790.210611:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516866816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210630:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575790.210633:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210635:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575790.210636:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210638:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575790.210639:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575790.210642:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575790.210646:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575790.210648:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575790.210651:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516866816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575790.210658:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516866816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575790.210661:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210663:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575790.210672:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516866560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210678:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575790.210680:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575790.210682:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575790.210685:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575790.210686:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575790.210687:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575790.210689:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575790.210691:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575790.210693:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575790.210694:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575790.210695:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575790.210699:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575790.210700:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210701:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575790.210704:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516866688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210708:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575790.210709:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575790.210710:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f000. 02000000:00000001:0.0:1788575790.210712:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575790.210712:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575790.210713:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575790.210714:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575790.210714:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575790.210716:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575790.210717:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575790.210717:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575790.210719:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575790.210720:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210721:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575790.210722:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516866816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210726:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575790.210730:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575790.210731:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575790.210733:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575790.210734:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575790.210734:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575790.210736:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575790.210736:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575790.210737:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575790.210738:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575790.210739:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575790.210741:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575790.210742:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210743:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210745:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575790.210746:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.210747:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575790.210748:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575790.210749:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575790.210751:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516866944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210755:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575790.210756:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575790.210758:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575790.210759:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575790.210760:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575790.210762:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516866944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575790.210764:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575790.210765:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575790.210766:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575790.210768:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575790.210769:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575790.210770:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575790.210773:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575790.210774:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575790.210776:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575790.210778:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516866944, portal 4 00000100:00000001:0.0:1788575790.210782:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575790.210783:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575790.210787:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516866944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210791:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575790.210792:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575790.210794:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575790.210795:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516866944, offset 0 00000400:00000200:0.0:1788575790.210798:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575790.210800:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575790.210802:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575790.210804:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575790.210805:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575790.210821:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575790.210823:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575790.210825:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548425:548425:256:4294967295] 192.168.202.128@tcp LPNI seq info [548348:548348:8:4294967295] 00000400:00000200:0.0:1788575790.210827:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575790.210831:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575790.210834:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575790.210836:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575790.210839:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575790.210840:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788575790.210847:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575790.210849:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575790.210850:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575790.210852:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575790.210854:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575790.210855:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575790.210857:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575790.210859:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516866944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210871:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575790.210873:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516866944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210877:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.210878:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575790.210880:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575790.210882:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575790.210885:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575790.210886:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311150249 00000100:00000001:0.0:1788575790.210888:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311153901 00000100:00000001:0.0:1788575790.210889:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311150249 00000100:00000001:0.0:1788575790.210890:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311153901 00000100:00000001:0.0:1788575790.210891:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575790.210892:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.210893:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575790.210895:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575790/real 1788575790] req@ffff8fc4901a4e00 x1875440516866944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575790.210899:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575790.210900:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575790.210901:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575790.210903:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575790.210904:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575790.210905:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575790.210907:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516866944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575790.210912:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516866944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210915:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575790.210919:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210921:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575790.210921:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575790.210924:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516866944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210928:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575790.210930:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516866944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210934:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575790.210936:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210937:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575790.210937:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210938:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575790.210939:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575790.210941:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575790.210942:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575790.210943:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575790.210945:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516866944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575790.210949:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516866944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575790.210951:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210952:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575790.210954:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516866944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575790.210958:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575790.210959:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575790.210960:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575790.210961:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575790.210962:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575790.210965:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575790.210966:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575790.210968:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575790.210969:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575790.210970:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575790.210971:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575790.210973:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575790.210974:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210976:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.210978:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575790.210979:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.210980:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575790.210981:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.720910:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575790.720918:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575790.720920:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575790.720927:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575790.720934:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575790.720939:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.720944:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575790.720948:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.720952:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575790.720977:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575790.720981:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575790.720982:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575790.720985:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575790.720987:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575790.720989:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.720991:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575790.720993:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575790.720995:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575791.424984:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575791.424988:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575791.424991:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575792.768935:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575792.768942:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575792.768947:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575792.768955:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575792.768974:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575792.768978:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575792.768991:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575792.768994:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575792.768997:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575793.473006:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575793.473010:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575793.473014:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575793.792940:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575793.792948:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575793.792952:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575793.792959:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575793.792966:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575793.792971:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575793.793057:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575793.793060:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575793.793063:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575794.048920:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311153901, at 4311153920 00000100:00000010:0.0:1788575794.048934:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575794.497073:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575794.497077:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575794.497081:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575794.816883:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575794.816890:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575794.816895:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575794.816904:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575794.816910:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575794.816913:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575794.816917:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575794.816920:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575794.816923:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575794.816976:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575794.816978:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575794.816979:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575794.816980:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575794.816983:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575794.816984:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575794.817000:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575794.817002:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575794.817004:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575795.328939:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575795.328950:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575795.328953:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575795.328956:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575795.328961:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575795.328968:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16482 00000400:00000200:0.0:1788575795.328977:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575795.328983:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575795.328986:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575795.328989:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.328992:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575795.328994:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.328996:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575795.329000:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575795.329003:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575795.329005:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575795.329008:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575795.329010:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575795.329011:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575795.329013:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329014:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575795.329016:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329017:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575795.329019:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575795.329021:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329022:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575795.329024:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575795.329027:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575795.329043:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329045:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575795.329047:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575795.329052:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575795.329054:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575795.329057:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575795.329059:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329061:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575795.329063:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575795.329066:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575795.329071:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575795.329076:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575795.329083:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516867072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575795.329090:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575795.329093:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575795.329096:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516867072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575795.329113:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575795.329115:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575795.329118:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575795.329119:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575795.329120:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575795.329122:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575795.329123:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16482 00000400:00000200:0.0:1788575795.329126:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575795.329128:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575795.329130:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575795.329132:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329134:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575795.329136:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329141:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575795.329143:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575795.329146:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575795.329148:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575795.329151:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575795.329154:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575795.329156:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575795.329160:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329161:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575795.329162:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329163:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575795.329164:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575795.329165:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329168:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575795.329169:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575795.329173:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575795.329177:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329181:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575795.329184:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575795.329187:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329191:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575795.329195:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575795.329198:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575795.329202:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575795.329203:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329207:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575795.329209:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575795.329212:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575795.329217:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575795.329222:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575795.329228:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516867200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575795.329245:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575795.329249:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575795.329253:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516867200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575795.329262:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575795.329264:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575795.329270:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575795.329272:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575795.329274:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575795.329276:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575795.329280:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16482 00000400:00000200:0.0:1788575795.329284:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575795.329287:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575795.329290:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575795.329293:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329295:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575795.329296:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329301:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575795.329304:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575795.329306:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575795.329308:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575795.329312:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575795.329314:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575795.329316:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575795.329319:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329323:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575795.329326:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329329:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575795.329338:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575795.329344:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329347:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575795.329352:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575795.329356:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575795.329359:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329363:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575795.329368:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575795.329371:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329375:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575795.329380:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575795.329386:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575795.329391:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575795.329395:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329399:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575795.329402:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575795.329406:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575795.329414:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575795.329420:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575795.329427:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516867328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575795.329449:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575795.329452:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575795.329459:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516867328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575795.329472:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575795.329488:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575795.329493:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575795.329496:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575795.329500:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575795.329504:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575795.329514:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16482 00000400:00000200:0.0:1788575795.329521:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575795.329527:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575795.329530:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575795.329536:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329539:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575795.329543:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329548:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575795.329551:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575795.329555:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575795.329557:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575795.329560:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575795.329565:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575795.329570:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575795.329576:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329580:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575795.329586:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329589:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575795.329594:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575795.329601:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329606:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575795.329611:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575795.329616:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575795.329618:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329624:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575795.329628:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575795.329633:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329637:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575795.329643:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575795.329648:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575795.329651:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575795.329658:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329661:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575795.329664:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575795.329667:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575795.329675:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575795.329681:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575795.329688:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516867456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575795.329702:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575795.329707:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575795.329714:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516867456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575795.329728:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575795.329733:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575795.329739:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575795.329741:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575795.329745:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575795.329751:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575795.329755:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16482 00000400:00000200:0.0:1788575795.329763:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575795.329771:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575795.329774:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575795.329780:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329783:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575795.329787:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329792:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575795.329796:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575795.329797:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575795.329800:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575795.329803:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575795.329862:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575795.329873:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575795.329882:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329888:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575795.329893:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329898:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575795.329904:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575795.329912:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575795.329918:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575795.329922:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575795.329926:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575795.329931:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329935:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575795.329938:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575795.329942:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329946:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575795.329950:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575795.329956:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575795.329961:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575795.329964:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575795.329966:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575795.329970:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575795.329973:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575795.329980:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575795.329986:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575795.329994:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516867584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575795.330003:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575795.330007:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575795.330013:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516867584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575795.330027:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575795.330037:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16492) 00000100:00000001:0.0:1788575795.840887:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575795.840894:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575795.840899:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575795.840905:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575795.840915:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575795.840920:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575795.840925:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575795.840928:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575795.840932:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575796.544970:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575796.544973:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575796.544975:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575796.864912:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575796.864918:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575796.864920:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575796.864931:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575796.864939:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575796.864944:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575796.864965:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575796.864970:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575796.864974:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575797.888913:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575797.888920:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575797.888924:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575797.888933:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575797.888939:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575797.888944:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575797.888949:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575797.888952:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575797.888956:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575798.593001:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575798.593005:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575798.593009:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575798.912935:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575798.912939:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575798.912944:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575798.912950:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575798.912967:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575798.912970:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575798.912984:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575798.912987:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575798.912989:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575799.168890:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311159021, at 4311159040 00000100:00000010:0.0:1788575799.168904:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfb40. 00000020:00000001:0.0:1788575799.617095:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575799.617099:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575799.617102:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575799.936923:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575799.936930:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575799.936933:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575799.936939:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575799.936944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575799.936947:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575799.936950:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575799.936954:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575799.936957:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575799.937302:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575799.937305:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575799.937306:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575799.937309:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575799.937311:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575799.937313:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575799.937314:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575799.937316:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575799.937318:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575800.449078:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788575800.449086:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575800.449104:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575800.449110:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575800.449115:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.449266:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575800.449272:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575800.449276:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575800.449284:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516867712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.449320:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575800.449324:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575800.449331:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575800.449343:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575800.449348:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575800.449354:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516867712:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575800.449360:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575800.449364:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575800.449366:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575800.449369:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575800.449374:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575800.449376:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575800.449383:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575800.449389:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575800.449398:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575800.449406:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516867712, portal 25 00000100:00000001:0.0:1788575800.449411:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575800.449414:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575800.449423:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516867712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.449430:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575800.449434:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575800.449438:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575800.449441:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516867712, offset 0 00000400:00000200:0.0:1788575800.449449:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575800.449460:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575800.449467:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575800.449475:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575800.449481:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575800.449485:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575800.449489:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575800.449501:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548431:548431:256:4294967295] 192.168.202.128@tcp LPNI seq info [548354:548354:8:4294967295] 00000400:00000200:0.0:1788575800.449509:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575800.449524:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575800.449534:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575800.449539:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575800.449546:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575800.449549:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575800.449565:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575800.449570:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575800.449577:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575800.449586:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575800.449594:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575800.449599:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575800.449606:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575800.449617:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516867712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.449635:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575800.449651:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516867712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.449679:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.449683:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575800.449690:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575800.449697:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575800.449702:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311164141 00000100:00000010:0.0:1788575800.449710:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50000. 00000100:00000001:0.0:1788575800.449719:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575800.449728:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311164141 00000100:00000010:0.0:1788575800.449733:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b500c0. 00000100:00000001:0.0:1788575800.449739:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575800.449745:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575800.449755:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.449760:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575800.449769:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575800/real 1788575800] req@ffff8fc4901a4e00 x1875440516867712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575800.449792:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575800.449796:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575800.449801:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575800.449806:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575800.449837:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575800.449844:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575800.449851:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516867712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575800.449867:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516867712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.449881:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575800.449885:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.449891:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575800.449894:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575800.449903:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516867712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.449919:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575800.449928:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516867712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.449944:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575800.449950:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.449954:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575800.449957:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.449962:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575800.449966:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575800.449971:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575800.449977:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575800.449988:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575800.449999:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516867712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575800.450018:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516867712:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575800.450026:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450031:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575800.450038:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516867712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450053:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575800.450056:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575800.450060:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575800.450067:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575800.450072:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575800.450075:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575800.450080:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575800.450085:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575800.450092:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575800.450095:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575800.450099:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575800.450107:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575800.450111:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450116:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450123:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575800.450125:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.450130:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575800.450132:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575800.450134:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575800.450140:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516867840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450150:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575800.450153:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575800.450157:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575800.450159:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575800.450166:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575800.450171:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516867840:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575800.450177:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575800.450179:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575800.450180:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575800.450182:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575800.450184:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575800.450185:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575800.450188:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575800.450190:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575800.450192:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575800.450195:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516867840, portal 10 00000100:00000001:0.0:1788575800.450196:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575800.450198:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575800.450202:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516867840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450206:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575800.450208:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575800.450209:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575800.450210:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516867840, offset 0 00000400:00000200:0.0:1788575800.450212:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575800.450215:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575800.450217:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575800.450219:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575800.450221:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575800.450223:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575800.450224:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575800.450226:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548432:548432:256:4294967295] 192.168.202.128@tcp LPNI seq info [548355:548355:8:4294967295] 00000400:00000200:0.0:1788575800.450229:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575800.450233:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575800.450236:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575800.450241:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575800.450243:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575800.450245:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575800.450252:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575800.450254:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575800.450255:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575800.450258:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575800.450260:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575800.450261:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575800.450263:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575800.450265:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516867840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450271:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575800.450272:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516867840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450277:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.450278:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575800.450280:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575800.450282:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575800.450285:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575800.450286:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311160339 00000100:00000001:0.0:1788575800.450288:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311164141 00000100:00000001:0.0:1788575800.450290:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311160339 00000100:00000001:0.0:1788575800.450291:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311164141 00000100:00000001:0.0:1788575800.450292:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575800.450293:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.450295:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575800.450297:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575800/real 1788575800] req@ffff8fc4901a6300 x1875440516867840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575800.450308:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575800.450309:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575800.450310:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575800.450312:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575800.450312:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575800.450314:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575800.450316:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516867840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575800.450321:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516867840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450324:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575800.450325:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450326:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575800.450327:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575800.450328:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516867840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450345:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575800.450348:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516867840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450353:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575800.450354:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450356:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575800.450356:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450358:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575800.450359:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575800.450360:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575800.450362:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575800.450363:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575800.450370:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516867840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575800.450375:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516867840:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575800.450379:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575800.450382:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516867968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450388:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575800.450390:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575800.450392:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575800.450394:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575800.450396:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575800.450399:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516867968:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575800.450402:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575800.450404:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575800.450405:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575800.450407:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575800.450410:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575800.450412:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575800.450416:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575800.450418:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575800.450421:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575800.450424:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516867968, portal 10 00000100:00000001:0.0:1788575800.450426:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575800.450428:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575800.450433:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516867968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450437:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575800.450438:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575800.450439:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575800.450441:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516867968, offset 0 00000400:00000200:0.0:1788575800.450447:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575800.450450:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575800.450452:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575800.450454:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575800.450456:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575800.450458:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575800.450459:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575800.450461:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548433:548433:256:4294967295] 192.168.202.128@tcp LPNI seq info [548356:548356:8:4294967295] 00000400:00000200:0.0:1788575800.450464:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575800.450468:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575800.450471:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575800.450473:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575800.450475:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575800.450477:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575800.450483:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575800.450485:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575800.450487:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575800.450489:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575800.450491:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575800.450492:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575800.450494:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575800.450496:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516867968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450501:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575800.450503:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516867968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450506:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.450507:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575800.450509:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575800.450511:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575800.450513:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575800.450517:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311160346 00000100:00000001:0.0:1788575800.450519:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311164141 00000100:00000001:0.0:1788575800.450521:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311160346 00000100:00000001:0.0:1788575800.450522:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311164141 00000100:00000001:0.0:1788575800.450523:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575800.450525:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.450526:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575800.450528:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575800/real 1788575800] req@ffff8fc4901a7480 x1875440516867968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575800.450533:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575800.450534:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575800.450535:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575800.450536:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575800.450537:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575800.450539:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575800.450540:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516867968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575800.450545:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516867968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450548:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575800.450549:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450550:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575800.450551:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575800.450553:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516867968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450556:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575800.450558:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516867968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450562:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575800.450566:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450567:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575800.450568:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450569:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575800.450571:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575800.450573:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575800.450575:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575800.450576:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575800.450579:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516867968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575800.450583:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516867968:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575800.450585:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575800.450587:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516868096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450590:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575800.450591:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575800.450592:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575800.450593:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575800.450595:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575800.450596:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516868096:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575800.450598:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575800.450600:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575800.450601:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575800.450602:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575800.450604:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575800.450605:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575800.450607:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575800.450609:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575800.450611:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575800.450613:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516868096, portal 4 00000100:00000001:0.0:1788575800.450618:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575800.450619:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575800.450625:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516868096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450631:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575800.450633:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575800.450637:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575800.450641:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516868096, offset 0 00000400:00000200:0.0:1788575800.450645:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575800.450648:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575800.450651:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575800.450654:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575800.450656:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575800.450658:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575800.450660:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575800.450663:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548434:548434:256:4294967295] 192.168.202.128@tcp LPNI seq info [548357:548357:8:4294967295] 00000400:00000200:0.0:1788575800.450667:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575800.450672:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575800.450675:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575800.450678:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575800.450681:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575800.450684:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575800.450693:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575800.450695:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575800.450698:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575800.450701:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575800.450705:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575800.450706:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575800.450709:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575800.450711:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516868096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450728:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575800.450731:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516868096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450736:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.450738:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575800.450742:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575800.450745:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575800.450748:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575800.450749:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311160486 00000100:00000001:0.0:1788575800.450751:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311164141 00000100:00000001:0.0:1788575800.450753:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311160486 00000100:00000001:0.0:1788575800.450755:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311164141 00000100:00000001:0.0:1788575800.450756:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575800.450758:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.450760:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575800.450763:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575800/real 1788575800] req@ffff8fc4901a4700 x1875440516868096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575800.450770:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575800.450771:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575800.450772:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575800.450774:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575800.450776:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575800.450778:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575800.450780:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516868096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575800.450787:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516868096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450792:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575800.450797:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450799:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575800.450800:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575800.450803:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516868096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450830:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575800.450834:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516868096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450840:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575800.450842:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450844:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575800.450845:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450847:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575800.450849:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575800.450851:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575800.450854:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575800.450856:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575800.450860:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516868096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575800.450868:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516868096:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575800.450872:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450873:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575800.450876:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516867840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450882:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575800.450883:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575800.450885:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575800.450887:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575800.450888:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575800.450893:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575800.450895:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575800.450896:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575800.450898:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575800.450899:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575800.450900:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575800.450904:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575800.450905:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450906:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575800.450908:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516867968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450912:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575800.450913:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575800.450914:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9d800. 02000000:00000001:0.0:1788575800.450915:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575800.450916:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575800.450921:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575800.450922:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575800.450923:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575800.450925:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575800.450925:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575800.450926:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575800.450929:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575800.450930:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450931:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575800.450933:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516868096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450936:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575800.450936:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575800.450937:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575800.450939:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575800.450939:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575800.450940:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575800.450941:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575800.450942:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575800.450943:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575800.450946:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575800.450947:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575800.450949:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575800.450950:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450951:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.450953:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575800.450954:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.450955:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575800.450956:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575800.450957:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575800.450959:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516868224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450964:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575800.450965:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575800.450966:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575800.450967:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575800.450969:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575800.450970:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516868224:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575800.450972:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575800.450973:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575800.450974:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575800.450975:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575800.450977:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575800.450978:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575800.450980:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575800.450982:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575800.450984:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575800.450986:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516868224, portal 4 00000100:00000001:0.0:1788575800.450987:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575800.450988:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575800.450992:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516868224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.450995:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575800.450997:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575800.451001:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575800.451002:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516868224, offset 0 00000400:00000200:0.0:1788575800.451005:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575800.451007:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575800.451010:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575800.451012:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575800.451013:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575800.451015:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575800.451016:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575800.451018:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548435:548435:256:4294967295] 192.168.202.128@tcp LPNI seq info [548358:548358:8:4294967295] 00000400:00000200:0.0:1788575800.451020:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575800.451024:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575800.451027:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575800.451029:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575800.451031:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575800.451033:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788575800.451040:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575800.451041:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575800.451042:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575800.451045:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575800.451046:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575800.451048:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575800.451049:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575800.451052:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516868224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.451056:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575800.451058:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516868224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575825 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.451062:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.451063:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575800.451068:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575800.451070:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575800.451072:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575800.451073:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311160489 00000100:00000001:0.0:1788575800.451075:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311164141 00000100:00000001:0.0:1788575800.451076:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311160489 00000100:00000001:0.0:1788575800.451078:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311164141 00000100:00000001:0.0:1788575800.451079:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575800.451081:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.451082:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575800.451086:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575800/real 1788575800] req@ffff8fc4901a5880 x1875440516868224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575800.451093:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575800.451094:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575800.451096:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575800.451098:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575800.451100:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575800.451101:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575800.451104:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516868224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575800.451112:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516868224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.451119:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575800.451120:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.451123:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575800.451124:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575800.451127:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516868224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.451132:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575800.451135:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516868224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.451143:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575800.451145:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.451146:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575800.451147:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.451148:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575800.451149:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575800.451151:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575800.451153:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575800.451154:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575800.451157:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516868224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575800.451162:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516868224:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575800.451163:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.451164:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575800.451166:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516868224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575825 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575800.451169:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575800.451170:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575800.451171:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9f400. 02000000:00000001:0.0:1788575800.451173:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575800.451174:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575800.451175:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575800.451176:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575800.451177:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575800.451179:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575800.451180:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575800.451181:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575800.451183:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575800.451184:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.451187:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.451189:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575800.451190:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.451192:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575800.451193:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575800.960922:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575800.960931:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575800.960937:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575800.960948:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575800.960959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575800.960965:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.960981:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575800.960987:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575800.960995:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575801.665008:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575801.665012:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575801.665016:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788575803.712993:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575803.712995:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575803.712997:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575804.032924:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575804.032928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575804.032932:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575804.032937:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575804.032942:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575804.032945:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575804.032963:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575804.032966:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575804.032969:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575804.288919:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311164141, at 4311164160 00000100:00000010:0.0:1788575804.288935:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50000. 00000020:00000001:0.0:1788575804.737051:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575804.737055:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575804.737057:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575805.056917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575805.056924:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575805.056927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575805.056946:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575805.056950:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575805.056954:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575805.056966:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575805.056969:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575805.056971:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575805.568896:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575805.568905:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575805.568908:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575805.568911:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575805.568916:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575805.568922:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16493 00000400:00000200:0.0:1788575805.568931:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575805.568937:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575805.568940:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575805.568942:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.568945:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575805.568948:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.568950:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575805.568954:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575805.568956:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575805.568959:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575805.568962:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575805.568965:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575805.568967:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575805.568969:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.568971:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575805.568972:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.568973:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575805.568975:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575805.568978:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575805.568996:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575805.568999:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575805.569003:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575805.569006:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569008:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575805.569011:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575805.569017:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575805.569021:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575805.569025:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575805.569027:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569030:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575805.569032:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575805.569037:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575805.569045:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575805.569051:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575805.569058:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516868352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575805.569065:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575805.569068:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575805.569072:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516868352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575805.569088:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575805.569090:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575805.569092:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575805.569094:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575805.569095:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575805.569096:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575805.569098:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16493 00000400:00000200:0.0:1788575805.569101:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575805.569103:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575805.569105:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575805.569107:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569115:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575805.569118:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569121:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575805.569124:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575805.569125:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575805.569127:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575805.569130:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575805.569132:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575805.569135:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575805.569140:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569143:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575805.569147:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569149:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575805.569153:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575805.569155:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569158:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575805.569162:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575805.569167:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575805.569171:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569174:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575805.569178:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575805.569181:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569184:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575805.569187:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575805.569201:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575805.569203:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575805.569204:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569205:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575805.569206:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575805.569207:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575805.569210:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575805.569213:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575805.569223:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516868480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575805.569233:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575805.569235:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575805.569238:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516868480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575805.569249:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575805.569253:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575805.569257:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575805.569258:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575805.569261:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575805.569263:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575805.569266:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16493 00000400:00000200:0.0:1788575805.569271:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575805.569274:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575805.569277:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575805.569281:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569283:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575805.569284:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569285:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575805.569289:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575805.569290:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575805.569292:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575805.569294:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575805.569298:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575805.569302:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575805.569307:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569310:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575805.569313:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569320:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575805.569324:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575805.569327:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569330:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575805.569332:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575805.569335:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575805.569337:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569339:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575805.569341:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575805.569344:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569347:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575805.569350:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575805.569353:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575805.569357:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575805.569358:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569359:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575805.569361:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575805.569363:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575805.569374:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575805.569379:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575805.569384:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516868608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575805.569394:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575805.569398:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575805.569407:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516868608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575805.569426:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575805.569432:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575805.569437:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575805.569440:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575805.569444:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575805.569456:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575805.569464:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16493 00000400:00000200:0.0:1788575805.569473:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575805.569481:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575805.569486:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575805.569491:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569494:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575805.569497:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569506:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575805.569511:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575805.569513:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575805.569516:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575805.569518:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575805.569521:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575805.569525:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575805.569531:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569532:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575805.569537:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569539:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575805.569541:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575805.569546:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569549:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575805.569552:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575805.569555:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575805.569558:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569561:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575805.569563:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575805.569565:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569568:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575805.569572:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575805.569579:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575805.569584:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575805.569587:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569589:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575805.569591:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575805.569594:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575805.569600:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575805.569604:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575805.569612:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516868736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575805.569625:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575805.569628:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575805.569634:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516868736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575805.569645:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575805.569650:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575805.569654:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575805.569656:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575805.569659:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575805.569663:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575805.569667:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16493 00000400:00000200:0.0:1788575805.569674:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575805.569679:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575805.569682:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575805.569687:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569690:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575805.569694:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569697:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575805.569700:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575805.569702:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575805.569708:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575805.569711:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575805.569715:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575805.569720:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575805.569727:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569729:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575805.569733:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569737:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575805.569740:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575805.569745:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575805.569749:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575805.569753:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575805.569756:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575805.569758:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569761:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575805.569769:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575805.569773:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569776:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575805.569780:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575805.569784:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575805.569787:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575805.569788:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575805.569789:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575805.569790:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575805.569791:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575805.569794:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575805.569797:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575805.569801:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516868864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575805.569845:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575805.569848:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575805.569853:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516868864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575805.569880:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575805.569886:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16503) 00000100:00000001:0.0:1788575806.080905:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575806.080914:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575806.080918:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575806.080925:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575806.080931:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575806.080935:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575806.080941:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575806.080945:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575806.080951:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575806.080987:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575806.080991:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575806.080992:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575806.080995:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575806.080997:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575806.080999:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575806.081001:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575806.081003:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575806.081006:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575806.784972:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575806.784976:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575806.784979:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575807.104909:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575807.104918:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575807.104923:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575807.104932:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575807.104939:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575807.104945:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575807.104951:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575807.104955:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575807.104961:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575807.104983:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575807.104987:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575807.104990:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575807.104992:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575807.105007:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575807.105009:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575807.105011:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575807.105012:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575807.105014:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575808.128899:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575808.128905:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575808.128909:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575808.128917:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575808.128923:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575808.128927:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575808.128945:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575808.128948:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575808.128950:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575808.129075:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575808.129077:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575808.129078:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575808.129080:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575808.129083:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575808.129085:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575808.129086:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575808.129087:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575808.129088:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575808.832961:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575808.832965:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575808.832968:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575809.152937:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575809.152956:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575809.152960:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575809.152966:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575809.152972:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575809.152977:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575809.152983:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575809.152984:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575809.152989:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575809.153166:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575809.153170:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575809.153173:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575809.153190:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575809.153193:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575809.153195:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575809.153197:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575809.153198:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575809.153200:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575809.408920:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311169261, at 4311169280 00000100:00000010:0.0:1788575809.408936:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf540. 00000020:00000001:0.0:1788575809.857029:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575809.857031:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575809.857034:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575810.176923:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575810.176932:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575810.176935:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575810.176941:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575810.176948:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575810.176953:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575810.176964:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575810.176968:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575810.176971:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575811.200919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575811.200928:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575811.200932:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575811.200938:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575811.200945:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575811.200950:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575811.200973:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575811.200977:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575811.200980:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575811.904988:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575811.904993:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575811.904996:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575812.224939:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575812.224943:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575812.224946:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575812.224952:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575812.224960:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575812.224970:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575812.224972:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575812.224974:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575812.224978:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575813.248948:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575813.248957:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575813.248962:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575813.248972:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575813.248991:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575813.248996:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575813.249010:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575813.249014:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575813.249019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575813.953032:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575813.953035:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575813.953038:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575814.272913:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575814.272920:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575814.272923:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575814.272929:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575814.272935:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575814.272940:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575814.272968:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575814.272972:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575814.272977:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575814.528914:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311174381, at 4311174400 00000100:00000010:0.0:1788575814.528927:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf780. 00000020:00000001:0.0:1788575814.977067:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575814.977070:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575814.977072:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788575815.809139:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575815.809147:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575815.809150:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575815.809154:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575815.809157:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575815.809163:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16503 00000400:00000200:0.0:1788575815.809184:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575815.809188:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575815.809191:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575815.809193:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809196:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575815.809198:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809200:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575815.809203:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575815.809204:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575815.809206:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575815.809208:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575815.809209:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575815.809211:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575815.809213:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809214:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575815.809215:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809216:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575815.809219:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575815.809221:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809222:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575815.809224:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575815.809226:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575815.809228:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.809229:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575815.809231:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575815.809235:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575815.809237:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575815.809240:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575815.809245:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.809247:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575815.809250:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575815.809253:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575815.809261:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575815.809266:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575815.809273:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516869632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575815.809286:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575815.809290:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575815.809295:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516869632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575815.809314:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575815.809317:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575815.809321:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575815.809323:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575815.809324:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575815.809327:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575815.809330:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16503 00000400:00000200:0.0:1788575815.809334:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575815.809338:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575815.809340:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575815.809343:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809344:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575815.809346:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809348:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575815.809351:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575815.809353:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575815.809356:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575815.809358:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575815.809361:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575815.809363:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575815.809367:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809377:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575815.809383:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809390:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575815.809396:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575815.809403:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809408:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575815.809413:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575815.809421:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575815.809425:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.809428:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575815.809434:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575815.809447:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.809450:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575815.809455:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575815.809461:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575815.809467:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575815.809470:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.809474:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575815.809476:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575815.809479:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575815.809486:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575815.809491:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575815.809498:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516869760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575815.809508:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575815.809510:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575815.809513:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516869760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575815.809533:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575815.809538:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575815.809545:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575815.809554:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575815.809558:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575815.809564:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575815.809569:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16503 00000400:00000200:0.0:1788575815.809582:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575815.809587:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575815.809589:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575815.809592:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809596:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575815.809597:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809599:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575815.809602:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575815.809604:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575815.809608:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575815.809611:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575815.809614:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575815.809617:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575815.809621:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809624:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575815.809627:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809630:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575815.809632:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575815.809636:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809640:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575815.809643:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575815.809658:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575815.809664:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.809668:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575815.809672:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575815.809678:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.809686:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575815.809691:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575815.809696:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575815.809702:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575815.809704:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.809709:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575815.809713:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575815.809716:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575815.809723:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575815.809729:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575815.809736:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516869888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575815.809746:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575815.809748:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575815.809754:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516869888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575815.809765:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575815.809774:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575815.809780:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575815.809784:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575815.809787:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575815.809791:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575815.809795:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16503 00000400:00000200:0.0:1788575815.809801:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575815.809849:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575815.809853:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575815.809860:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809865:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575815.809870:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809877:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575815.809882:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575815.809891:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575815.809895:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575815.809898:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575815.809901:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575815.809904:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575815.809920:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809923:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575815.809927:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809930:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575815.809933:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575815.809938:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575815.809945:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575815.809949:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575815.809954:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575815.809959:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.809963:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575815.809966:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575815.809972:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.809976:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575815.809981:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575815.809985:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575815.809989:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575815.809992:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.809995:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575815.809998:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575815.810000:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575815.810006:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575815.810010:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575815.810017:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516870016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575815.810034:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575815.810038:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575815.810048:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516870016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575815.810066:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575815.810069:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575815.810074:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575815.810078:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575815.810079:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575815.810085:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575815.810088:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16503 00000400:00000200:0.0:1788575815.810093:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575815.810097:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575815.810099:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575815.810103:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.810107:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575815.810110:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.810113:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575815.810116:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575815.810119:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575815.810121:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575815.810123:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575815.810126:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575815.810129:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575815.810131:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.810132:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575815.810136:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575815.810141:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575815.810144:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575815.810148:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575815.810154:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575815.810160:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575815.810164:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575815.810169:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.810173:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575815.810177:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575815.810181:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.810184:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575815.810187:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575815.810189:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575815.810194:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575815.810196:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575815.810198:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575815.810199:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575815.810201:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575815.810204:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575815.810206:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575815.810210:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516870144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575815.810218:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575815.810223:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575815.810231:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516870144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575815.810251:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575815.810260:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16513) 00000100:00000001:0.0:1788575816.320940:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575816.320946:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575816.320952:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575816.320959:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575816.320968:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575816.320975:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575816.320985:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575816.320990:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575816.320995:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575817.025024:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575817.025045:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575817.025048:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575817.344986:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575817.344990:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575817.344993:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575817.344996:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575817.344999:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575817.345003:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575817.345005:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575817.345008:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575817.345014:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575818.368937:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575818.368943:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575818.368948:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575818.368954:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575818.368959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575818.368963:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575818.368994:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575818.368998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575818.369003:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575819.072983:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575819.072986:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575819.072988:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575819.393048:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575819.393052:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575819.393055:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575819.393060:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575819.393065:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575819.393070:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575819.393078:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575819.393081:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575819.393085:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575819.393690:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575819.393693:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575819.393694:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575819.393697:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575819.393699:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575819.393700:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575819.393714:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575819.393715:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575819.393717:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575819.648912:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311179501, at 4311179520 00000100:00000010:0.0:1788575819.648929:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfc60. 00000020:00000001:0.0:1788575820.097011:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575820.097015:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575820.097018:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575820.416931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575820.416938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575820.416941:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575820.416945:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575820.416949:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575820.416954:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575820.416957:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575820.416960:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575820.416964:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575820.416993:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575820.416997:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575820.417001:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575820.417004:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575820.417008:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575820.417014:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575820.417018:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575820.417022:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575820.417026:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575821.440942:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575821.440949:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575821.440951:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575821.440957:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575821.440962:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575821.440967:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575821.440971:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575821.440976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575821.440981:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575821.441004:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575821.441009:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575821.441025:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575821.441033:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575821.441043:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575821.441049:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575821.441055:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575821.441068:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575821.441073:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575822.144990:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575822.144993:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575822.144995:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575822.464921:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575822.464929:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575822.464933:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575822.464945:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575822.464952:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575822.464958:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575822.464992:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575822.464997:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575822.465002:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.488890:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575823.488897:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575823.488901:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575823.488909:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575823.488917:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575823.488924:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575823.488942:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575823.488946:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575823.488951:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575823.488971:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575823.488975:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575823.488977:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575823.488979:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575823.488982:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575823.488985:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575823.488988:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575823.488990:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575823.488993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575824.193006:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575824.193015:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575824.193017:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575824.512933:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575824.512939:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575824.512942:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575824.512949:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575824.512956:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575824.512961:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575824.512976:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575824.512979:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575824.512982:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575824.513148:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575824.513151:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575824.513153:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575824.513156:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575824.513158:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575824.513160:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575824.513162:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575824.513163:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575824.513165:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575824.768894:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311184621, at 4311184640 00000100:00000010:0.0:1788575824.768905:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf9c0. 00000020:00000001:0.0:1788575825.217061:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575825.217065:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575825.217068:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575825.536912:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575825.536918:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575825.536920:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575825.536925:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575825.536928:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575825.536932:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575825.536934:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575825.536937:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575825.536939:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575826.048950:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575826.048960:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575826.048978:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575826.048982:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575826.048986:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575826.048993:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16513 00000400:00000200:0.0:1788575826.049001:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575826.049006:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575826.049009:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575826.049012:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049015:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575826.049017:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049020:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575826.049024:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575826.049025:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575826.049028:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575826.049030:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575826.049033:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575826.049035:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575826.049038:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049040:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575826.049042:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049044:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575826.049047:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575826.049050:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049052:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575826.049054:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575826.049057:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575826.049060:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.049062:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575826.049064:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575826.049070:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575826.049074:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575826.049081:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575826.049082:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.049085:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575826.049088:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575826.049091:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575826.049098:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575826.049103:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575826.049111:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516870912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575826.049121:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575826.049125:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575826.049130:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516870912/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575826.049149:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575826.049152:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575826.049155:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575826.049157:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575826.049159:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575826.049162:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575826.049165:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16513 00000400:00000200:0.0:1788575826.049170:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575826.049173:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575826.049178:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575826.049183:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049188:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575826.049193:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049196:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575826.049201:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575826.049203:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575826.049207:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575826.049210:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575826.049221:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575826.049226:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575826.049233:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049237:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575826.049242:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049256:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575826.049264:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575826.049270:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049275:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575826.049278:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575826.049285:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575826.049289:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.049291:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575826.049297:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575826.049302:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.049307:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575826.049313:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575826.049319:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575826.049324:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575826.049327:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.049332:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575826.049335:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575826.049347:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575826.049357:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575826.049364:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575826.049373:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516871040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575826.049387:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575826.049391:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575826.049399:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516871040/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575826.049435:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575826.049441:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575826.049447:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575826.049450:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575826.049454:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575826.049456:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575826.049461:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16513 00000400:00000200:0.0:1788575826.049469:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575826.049487:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575826.049496:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575826.049503:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049508:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575826.049512:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049515:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575826.049520:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575826.049522:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575826.049525:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575826.049529:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575826.049532:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575826.049535:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575826.049541:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049546:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575826.049550:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049555:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575826.049560:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575826.049566:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049573:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575826.049582:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575826.049588:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575826.049593:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.049595:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575826.049599:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575826.049603:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.049604:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575826.049609:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575826.049613:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575826.049617:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575826.049631:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.049637:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575826.049639:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575826.049642:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575826.049648:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575826.049657:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575826.049665:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516871168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575826.049674:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575826.049678:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575826.049683:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516871168/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575826.049697:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575826.049701:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575826.049705:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575826.049709:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575826.049718:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575826.049723:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575826.049728:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16513 00000400:00000200:0.0:1788575826.049732:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575826.049738:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575826.049743:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575826.049754:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049757:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575826.049759:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049761:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575826.049764:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575826.049767:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575826.049770:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575826.049773:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575826.049778:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575826.049782:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575826.049790:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049795:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575826.049799:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049803:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575826.049828:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575826.049832:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575826.049834:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575826.049836:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575826.049840:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575826.049846:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.049850:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575826.049855:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575826.049860:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.049864:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575826.049868:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575826.049873:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575826.049880:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575826.049883:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.049886:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575826.049889:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575826.049893:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575826.049899:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575826.049914:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575826.049925:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516871296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575826.049940:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575826.049943:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575826.049952:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516871296/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575826.049964:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575826.049969:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575826.049976:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575826.049979:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575826.049982:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575826.049989:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575826.049995:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16513 00000400:00000200:0.0:1788575826.050002:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575826.050009:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575826.050014:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575826.050021:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.050026:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575826.050030:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.050034:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575826.050037:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575826.050040:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575826.050042:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575826.050044:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575826.050047:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575826.050053:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575826.050055:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.050058:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575826.050064:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575826.050067:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575826.050070:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575826.050072:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575826.050074:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575826.050075:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575826.050077:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575826.050082:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.050084:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575826.050087:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575826.050089:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.050093:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575826.050098:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575826.050101:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575826.050106:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575826.050108:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575826.050112:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575826.050115:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575826.050118:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575826.050124:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575826.050131:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575826.050138:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516871424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575826.050154:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575826.050158:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575826.050166:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516871424/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575826.050183:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575826.050188:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16523) 00000100:00000001:0.0:1788575826.560957:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575826.560962:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575826.560965:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575826.560977:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575826.560983:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575826.560987:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575826.560991:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575826.560995:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575826.560999:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575827.265011:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575827.265015:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575827.265019:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788575829.312986:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575829.312989:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575829.312991:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575829.633049:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575829.633052:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575829.633054:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575829.633057:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575829.633059:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575829.633062:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575829.633132:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575829.633134:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575829.633136:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575829.888906:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311189741, at 4311189760 00000100:00000010:0.0:1788575829.888921:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf6c0. 00000020:00000001:0.0:1788575830.337018:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575830.337022:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575830.337025:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575830.656999:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575830.657009:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575830.657016:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575830.657022:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575830.657027:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575830.657031:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575830.657037:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575830.657039:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575830.657044:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575831.680907:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575831.680915:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575831.680932:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575831.680938:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575831.680944:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575831.680949:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575831.680975:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575831.680984:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575831.680990:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575832.385061:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575832.385066:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575832.385070:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575832.704963:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575832.704973:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575832.704977:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575832.704985:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575832.704994:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575832.705001:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575832.705007:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575832.705015:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575832.705024:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575832.705052:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575832.705057:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575832.705061:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575832.705064:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575832.705068:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575832.705071:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575832.705075:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575832.705078:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575832.705080:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575833.728954:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575833.728961:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575833.728965:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575833.728971:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575833.728982:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575833.728997:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575833.729039:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575833.729043:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575833.729049:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575834.433005:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575834.433013:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575834.433015:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575834.752940:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575834.752944:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575834.752947:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575834.752950:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575834.752955:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575834.752958:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575834.752977:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575834.752981:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575834.752985:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575835.008922:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311194861, at 4311194880 00000100:00000010:0.0:1788575835.008939:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf600. 00000020:00000001:0.0:1788575835.457017:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575835.457020:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575835.457022:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575835.776925:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575835.776931:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575835.776934:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575835.776942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575835.776949:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575835.776954:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575835.776963:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575835.776967:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575835.776973:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575836.288978:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575836.288988:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575836.288991:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575836.288995:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575836.288999:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575836.289007:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16523 00000400:00000200:0.0:1788575836.289015:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575836.289020:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575836.289023:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575836.289026:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289041:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575836.289044:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289047:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575836.289050:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575836.289052:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575836.289054:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575836.289065:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575836.289068:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575836.289070:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575836.289073:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289075:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575836.289077:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289078:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575836.289081:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575836.289084:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289086:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575836.289088:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575836.289092:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575836.289094:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289096:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575836.289099:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575836.289105:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575836.289108:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575836.289111:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575836.289113:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289116:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575836.289119:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575836.289123:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575836.289131:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575836.289137:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575836.289146:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516872192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575836.289164:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575836.289167:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575836.289171:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516872192/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575836.289187:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575836.289190:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575836.289194:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575836.289195:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575836.289197:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575836.289200:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575836.289202:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16523 00000400:00000200:0.0:1788575836.289207:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575836.289210:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575836.289213:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575836.289216:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289218:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575836.289220:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289223:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575836.289228:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575836.289231:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575836.289233:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575836.289237:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575836.289241:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575836.289244:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575836.289253:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289258:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575836.289262:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289267:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575836.289282:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575836.289291:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289295:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575836.289299:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575836.289307:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575836.289312:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289317:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575836.289323:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575836.289329:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289334:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575836.289370:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575836.289375:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575836.289380:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575836.289383:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289389:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575836.289392:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575836.289394:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575836.289401:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575836.289407:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575836.289416:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516872320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575836.289429:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575836.289433:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575836.289440:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516872320/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575836.289452:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575836.289458:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575836.289464:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575836.289469:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575836.289473:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575836.289478:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575836.289489:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16523 00000400:00000200:0.0:1788575836.289501:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575836.289508:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575836.289512:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575836.289516:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289521:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575836.289525:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289527:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575836.289530:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575836.289531:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575836.289533:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575836.289535:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575836.289537:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575836.289539:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575836.289542:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289544:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575836.289545:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289548:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575836.289550:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575836.289552:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289555:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575836.289557:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575836.289560:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575836.289562:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289564:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575836.289567:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575836.289569:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289570:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575836.289574:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575836.289576:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575836.289578:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575836.289584:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289586:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575836.289588:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575836.289590:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575836.289593:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575836.289597:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575836.289602:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516872448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575836.289609:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575836.289611:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575836.289614:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516872448/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575836.289622:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575836.289625:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575836.289629:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575836.289630:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575836.289632:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575836.289635:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575836.289638:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16523 00000400:00000200:0.0:1788575836.289642:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575836.289645:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575836.289647:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575836.289649:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289651:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575836.289653:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289655:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575836.289656:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575836.289658:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575836.289659:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575836.289661:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575836.289663:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575836.289670:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575836.289674:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289676:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575836.289677:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289680:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575836.289682:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575836.289684:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289687:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575836.289688:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575836.289691:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575836.289693:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289695:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575836.289697:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575836.289699:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289700:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575836.289702:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575836.289704:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575836.289706:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575836.289706:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289708:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575836.289709:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575836.289710:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575836.289712:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575836.289715:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575836.289718:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516872576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575836.289723:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575836.289724:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575836.289727:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516872576/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575836.289731:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575836.289736:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575836.289739:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575836.289740:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575836.289741:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575836.289743:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575836.289745:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16523 00000400:00000200:0.0:1788575836.289748:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575836.289750:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575836.289752:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575836.289754:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289757:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575836.289758:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289761:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575836.289763:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575836.289764:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575836.289765:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575836.289767:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575836.289769:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575836.289771:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575836.289774:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289776:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575836.289778:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289780:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575836.289782:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575836.289785:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575836.289787:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575836.289789:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575836.289791:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575836.289793:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289795:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575836.289803:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575836.289805:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289806:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575836.289845:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575836.289846:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575836.289848:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575836.289849:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575836.289851:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575836.289852:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575836.289853:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575836.289855:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575836.289858:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575836.289862:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516872704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575836.289867:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575836.289868:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575836.289872:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516872704/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575836.289880:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575836.289884:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16533) 00000100:00000001:0.0:1788575836.800907:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575836.800915:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575836.800920:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575836.800929:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575836.800937:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575836.800944:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575836.800959:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575836.800963:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575836.800968:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575837.504979:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575837.504982:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575837.504984:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575837.824902:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575837.824906:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575837.824925:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575837.824930:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575837.824934:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575837.824937:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575837.824939:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575837.824940:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575837.824943:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575837.828394:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575837.828397:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575837.828398:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575837.828401:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575837.828403:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575837.828405:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575837.828407:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575837.828408:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575837.828410:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575838.849020:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575838.849023:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575838.849024:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575838.849026:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575838.849028:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575838.849030:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575838.849032:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575838.849033:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575838.849035:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575839.361119:0:2391:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1788575839.553097:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575839.553101:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575839.553104:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575839.872903:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575839.872909:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575839.872914:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575839.872921:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575839.872927:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575839.872931:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575839.872988:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575839.872991:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575839.872994:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575840.128929:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311199982, at 4311200000 00000100:00000010:0.0:1788575840.128947:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfd80. 00000020:00000001:0.0:1788575840.577085:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575840.577088:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575840.577091:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575840.896928:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575840.896934:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575840.896938:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575840.896944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575840.896948:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575840.896953:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575840.897192:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575840.897195:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575840.897198:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575841.921486:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575841.921492:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575841.921495:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575841.921498:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575841.921501:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575841.921503:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575841.921505:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575841.921507:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575841.921510:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575842.625028:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575842.625032:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575842.625035:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575842.947341:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575842.947344:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575842.947345:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575842.947348:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575842.947350:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575842.947352:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575842.947354:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575842.947355:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575842.947357:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575843.969005:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575843.969008:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575843.969026:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575843.969029:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575843.969031:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575843.969033:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575843.969035:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575843.969036:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575843.969038:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575843.969736:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575843.969740:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575843.969741:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575843.969749:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575843.969754:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575843.969758:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575843.969765:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575843.969769:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575843.969772:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575844.673064:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575844.673068:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575844.673071:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575844.992917:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575844.992926:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575844.992929:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575844.992934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575844.992939:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575844.992943:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575844.992946:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575844.992949:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575844.992952:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575844.992987:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575844.992991:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575844.992993:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575844.992995:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575844.992998:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575844.993002:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575844.993006:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575844.993010:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575844.993015:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575845.249023:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311205102, at 4311205120 00000100:00000010:0.0:1788575845.249044:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf4e0. 00000020:00000001:0.0:1788575845.696997:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575845.697001:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575845.697004:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575846.016919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575846.016928:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575846.016932:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575846.016939:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575846.016944:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575846.016948:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575846.016951:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575846.016954:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575846.016958:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.016981:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575846.016985:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575846.016987:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575846.016991:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575846.016993:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575846.016995:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575846.017003:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575846.017005:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575846.017008:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575846.529578:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575846.529586:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575846.529588:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575846.529591:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575846.529594:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575846.529600:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16533 00000400:00000200:0.0:1788575846.529606:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575846.529611:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575846.529613:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575846.529616:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529618:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575846.529620:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529634:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575846.529637:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575846.529639:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575846.529641:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575846.529643:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575846.529644:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575846.529646:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575846.529648:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529650:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575846.529651:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529652:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575846.529653:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575846.529655:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529657:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575846.529659:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575846.529661:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575846.529662:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.529664:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575846.529665:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575846.529669:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575846.529672:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575846.529674:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575846.529675:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.529677:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575846.529679:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575846.529683:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575846.529688:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575846.529694:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575846.529700:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516873472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575846.529707:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575846.529711:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575846.529717:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516873472/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575846.529737:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575846.529740:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575846.529742:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575846.529743:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575846.529744:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575846.529745:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575846.529747:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16533 00000400:00000200:0.0:1788575846.529750:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575846.529752:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575846.529753:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575846.529755:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529756:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575846.529757:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529759:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575846.529760:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575846.529761:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575846.529762:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575846.529764:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575846.529765:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575846.529766:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575846.529767:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529768:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575846.529769:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529770:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575846.529771:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575846.529773:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529774:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575846.529779:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575846.529784:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575846.529785:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.529786:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575846.529789:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575846.529791:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.529792:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575846.529794:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575846.529796:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575846.529797:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575846.529798:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.529799:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575846.529800:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575846.529801:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575846.529804:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575846.529806:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575846.529835:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516873600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575846.529840:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575846.529842:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575846.529844:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516873600/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575846.529849:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575846.529850:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575846.529853:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575846.529854:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575846.529855:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575846.529857:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575846.529858:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16533 00000400:00000200:0.0:1788575846.529860:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575846.529862:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575846.529864:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575846.529872:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529874:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575846.529875:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529878:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575846.529880:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575846.529881:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575846.529882:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575846.529884:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575846.529886:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575846.529888:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575846.529891:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529893:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575846.529895:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529897:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575846.529899:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575846.529901:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529904:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575846.529906:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575846.529909:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575846.529911:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.529913:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575846.529915:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575846.529917:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.529919:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575846.529922:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575846.529923:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575846.529925:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575846.529926:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.529927:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575846.529928:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575846.529930:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575846.529932:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575846.529939:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575846.529942:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516873728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575846.529947:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575846.529948:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575846.529951:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516873728/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575846.529955:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575846.529957:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575846.529960:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575846.529961:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575846.529962:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575846.529963:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575846.529964:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16533 00000400:00000200:0.0:1788575846.529967:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575846.529968:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575846.529970:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575846.529971:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529973:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575846.529974:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529975:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575846.529977:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575846.529978:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575846.529978:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575846.529980:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575846.529981:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575846.529982:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575846.529983:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529984:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575846.529988:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529990:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575846.529991:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575846.529993:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575846.529995:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575846.529996:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575846.529997:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575846.529998:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.529999:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575846.530001:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575846.530002:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.530003:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575846.530005:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575846.530007:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575846.530008:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575846.530009:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.530010:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575846.530011:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575846.530012:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575846.530015:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575846.530017:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575846.530020:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516873856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575846.530023:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575846.530025:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575846.530027:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516873856/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575846.530032:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575846.530033:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575846.530035:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575846.530035:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575846.530039:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575846.530041:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575846.530042:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16533 00000400:00000200:0.0:1788575846.530045:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575846.530047:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575846.530048:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575846.530050:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.530051:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575846.530052:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.530053:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575846.530055:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575846.530055:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575846.530056:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575846.530057:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575846.530058:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575846.530068:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575846.530071:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.530072:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575846.530073:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575846.530074:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575846.530075:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575846.530076:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575846.530077:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575846.530079:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575846.530081:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575846.530082:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.530083:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575846.530085:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575846.530087:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.530088:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575846.530093:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575846.530095:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575846.530096:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575846.530097:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575846.530098:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575846.530099:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575846.530101:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575846.530103:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575846.530105:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575846.530108:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516873984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575846.530113:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575846.530114:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575846.530116:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516873984/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575846.530121:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575846.530124:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16544) 00000100:00000001:0.0:1788575847.040949:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575847.040954:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575847.040956:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575847.040959:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575847.040963:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575847.040966:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575847.040970:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575847.040973:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575847.040978:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575847.747163:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575847.747167:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575847.747170:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575848.065174:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575848.065178:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575848.065179:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575848.065182:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575848.065185:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575848.065187:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575848.065205:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575848.065206:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575848.065209:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575848.066265:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575848.066267:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575848.066268:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575848.066271:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575848.066273:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575848.066275:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575848.066277:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575848.066278:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575848.066280:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575848.769066:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575848.769069:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575848.769073:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575849.089159:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575849.089164:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575849.089168:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575849.089174:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575849.089177:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575849.089183:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575849.089186:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575849.089190:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575849.089197:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575849.794462:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575849.794466:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575849.794469:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575850.113005:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575850.113011:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575850.113013:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575850.113017:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575850.113020:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575850.113023:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575850.113025:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575850.113028:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575850.113040:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575850.113055:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575850.113058:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575850.113068:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575850.113071:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575850.113073:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575850.113076:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575850.113079:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575850.113083:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575850.113090:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575850.368951:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311210225, at 4311210240 00000100:00000010:0.0:1788575850.368959:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfa80. 00000020:00000001:0.0:1788575850.816993:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575850.816996:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575850.816999:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575851.136915:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575851.136919:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575851.136922:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575851.136927:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575851.136931:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575851.136933:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575851.136944:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575851.136948:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575851.136951:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.136980:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575851.136984:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575851.136988:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575851.136991:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575851.136993:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575851.136996:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575851.136998:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575851.137001:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575851.137003:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575851.648897:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575851.648908:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575851.648911:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575851.648915:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575851.648920:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575851.648927:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16539 00000400:00000200:0.0:1788575851.648953:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575851.648980:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575851.648983:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575851.648985:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.648991:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575851.648993:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.648995:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575851.648999:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575851.649002:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575851.649004:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575851.649007:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575851.649009:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575851.649012:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575851.649014:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649016:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575851.649019:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649021:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575851.649024:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575851.649027:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649030:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575851.649033:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575851.649036:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575851.649039:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.649042:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575851.649043:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575851.649048:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575851.649051:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575851.649054:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575851.649055:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.649058:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575851.649060:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575851.649064:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575851.649073:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575851.649078:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575851.649086:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516874112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575851.649092:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575851.649095:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575851.649098:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516874112/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575851.649111:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575851.649113:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575851.649116:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575851.649117:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575851.649118:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575851.649121:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575851.649123:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16539 00000400:00000200:0.0:1788575851.649127:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575851.649129:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575851.649132:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575851.649134:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649137:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575851.649140:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649144:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575851.649148:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575851.649149:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575851.649151:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575851.649154:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575851.649158:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575851.649162:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575851.649166:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649174:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575851.649179:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649183:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575851.649187:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575851.649191:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649194:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575851.649196:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575851.649202:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575851.649207:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.649210:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575851.649214:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575851.649219:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.649221:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575851.649224:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575851.649228:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575851.649230:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575851.649234:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.649240:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575851.649244:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575851.649247:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575851.649255:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575851.649261:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575851.649271:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516874240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575851.649284:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575851.649286:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575851.649292:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516874240/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575851.649302:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575851.649305:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575851.649307:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575851.649314:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575851.649317:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575851.649321:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575851.649325:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16539 00000400:00000200:0.0:1788575851.649334:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575851.649341:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575851.649342:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575851.649344:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649346:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575851.649357:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649365:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575851.649369:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575851.649371:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575851.649373:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575851.649375:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575851.649378:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575851.649382:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575851.649387:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649391:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575851.649396:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649399:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575851.649402:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575851.649408:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649411:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575851.649412:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575851.649415:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575851.649418:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.649421:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575851.649425:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575851.649427:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.649433:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575851.649438:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575851.649443:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575851.649450:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575851.649453:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.649457:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575851.649461:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575851.649465:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575851.649473:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575851.649481:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575851.649490:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516874368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575851.649497:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575851.649501:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575851.649505:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516874368/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575851.649516:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575851.649521:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575851.649525:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575851.649529:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575851.649532:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575851.649537:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575851.649544:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16539 00000400:00000200:0.0:1788575851.649553:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575851.649558:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575851.649562:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575851.649569:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649571:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575851.649575:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649580:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575851.649583:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575851.649589:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575851.649593:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575851.649595:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575851.649598:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575851.649601:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575851.649607:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649612:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575851.649615:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649618:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575851.649622:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575851.649626:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649630:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575851.649633:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575851.649638:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575851.649642:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.649646:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575851.649650:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575851.649656:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.649663:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575851.649670:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575851.649674:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575851.649681:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575851.649684:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.649689:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575851.649693:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575851.649697:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575851.649714:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575851.649720:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575851.649730:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516874496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575851.649747:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575851.649751:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575851.649769:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516874496/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575851.649802:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575851.649836:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575851.649846:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575851.649849:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575851.649851:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575851.649856:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575851.649861:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16539 00000400:00000200:0.0:1788575851.649870:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575851.649877:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575851.649882:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575851.649888:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649891:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575851.649895:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649900:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575851.649904:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575851.649906:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575851.649909:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575851.649912:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575851.649918:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575851.649923:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575851.649931:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649937:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575851.649944:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649949:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575851.649966:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575851.649971:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575851.649975:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575851.649984:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575851.649988:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575851.649992:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.649995:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575851.650000:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575851.650007:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.650012:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575851.650019:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575851.650025:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575851.650030:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575851.650033:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575851.650036:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575851.650039:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575851.650044:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575851.650051:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575851.650059:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575851.650069:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516874624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575851.650085:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575851.650089:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575851.650094:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516874624/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575851.650106:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575851.650110:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16549) 00000100:00000001:0.0:1788575852.160913:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575852.160918:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575852.160921:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575852.160928:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575852.160934:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575852.160940:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575852.160967:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575852.160972:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575852.160979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575852.865076:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575852.865094:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575852.865098:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575853.184957:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575853.184967:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575853.184971:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575853.184980:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575853.184996:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575853.185000:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575853.185004:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575853.185006:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575853.185009:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575853.888986:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575853.888991:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575853.888994:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575854.208936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575854.208945:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575854.208948:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575854.208955:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575854.208961:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575854.208967:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575854.208973:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575854.208977:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575854.208980:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575854.913028:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575854.913031:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575854.913034:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575855.233127:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575855.233131:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575855.233132:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575855.233134:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575855.233136:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575855.233138:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575855.233316:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575855.233318:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575855.233321:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575855.488924:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311215341, at 4311215360 00000100:00000010:0.0:1788575855.488944:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf8a0. 00000020:00000001:0.0:1788575855.937084:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575855.937089:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575855.937092:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575856.257014:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575856.257024:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575856.257029:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575856.257044:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575856.257051:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575856.257056:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575856.257059:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575856.257061:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575856.257064:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575856.257227:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575856.257231:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575856.257236:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575856.257240:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575856.257243:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575856.257245:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575856.257248:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575856.257249:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575856.257252:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575857.985096:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575857.985100:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575857.985104:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788575859.009104:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575859.009108:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575859.009112:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575859.328943:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575859.328948:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575859.328950:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575859.328953:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575859.328956:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575859.328961:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575859.328967:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575859.328971:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575859.328975:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575859.329183:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575859.329186:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575859.329193:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575859.329195:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575859.329197:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575859.329199:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575859.329201:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575859.329202:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575859.329204:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575860.032996:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575860.033000:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575860.033003:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575860.352937:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575860.352945:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575860.352949:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575860.352959:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575860.352966:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575860.352974:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575860.353242:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575860.353245:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575860.353248:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575860.608924:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311220464, at 4311220480 00000100:00000010:0.0:1788575860.608940:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfd20. 00000020:00000001:0.0:1788575861.057004:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575861.057008:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575861.057012:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575861.377079:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575861.377083:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575861.377087:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575861.377091:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575861.377097:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575861.377101:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575861.379830:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575861.379837:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575861.382462:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575862.400942:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575862.400947:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575862.400951:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575862.400957:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575862.400978:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575862.400980:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575862.400983:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575862.400985:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575862.400988:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575862.402179:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575862.402184:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575862.402186:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575862.402189:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575862.402192:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575862.402194:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575862.402196:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575862.402198:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575862.402201:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575863.105912:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575863.105915:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575863.105919:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575863.424951:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575863.424959:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575863.424962:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575863.424969:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575863.424974:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575863.424979:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575863.424985:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575863.424988:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575863.424994:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575863.425454:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575863.425457:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575863.425459:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575863.425461:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575863.425464:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575863.425466:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575863.425468:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575863.425469:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575863.425472:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575864.129011:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575864.129014:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575864.129016:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575864.448906:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575864.448914:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575864.448918:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575864.448926:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575864.448931:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575864.448936:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575864.448940:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575864.448943:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575864.448946:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575864.448975:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575864.448979:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575864.448981:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575864.448987:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575864.448990:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575864.448994:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575864.449010:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575864.449013:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575864.449016:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575864.449029:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575864.449032:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575864.449033:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575864.449035:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575864.449037:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575864.449039:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575864.449041:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575864.449043:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575864.449045:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575865.153649:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575865.153653:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575865.153656:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575865.472938:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575865.472943:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575865.472946:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575865.472952:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575865.472956:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575865.472959:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575865.474220:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575865.474222:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575865.474253:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575865.728928:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311225581, at 4311225600 00000100:00000010:0.0:1788575865.728946:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf600. 00000020:00000001:0.0:1788575866.177050:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575866.177055:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575866.177058:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575866.497131:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575866.497136:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575866.497140:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575866.497147:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575866.497152:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575866.497157:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575866.497164:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575866.497167:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575866.497170:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575867.521036:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575867.521039:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575867.521040:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575867.521042:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575867.521044:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575867.521046:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575867.521047:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575867.521049:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575867.521051:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575867.521074:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575867.521078:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575867.521081:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575867.521084:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575867.521089:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575867.521091:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575867.521096:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575867.521098:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575867.521100:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575868.225006:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575868.225010:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575868.225020:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575868.544936:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575868.544966:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575868.544969:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575868.544972:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575868.544975:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575868.544977:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575868.544979:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575868.544981:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575868.544983:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575868.545164:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575868.545167:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575868.545169:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575868.545172:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575868.545175:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575868.545178:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575868.545181:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575868.545184:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575868.545186:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575869.248957:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575869.248961:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575869.248965:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788575870.273051:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575870.273054:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575870.273056:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575870.593199:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575870.593204:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575870.593207:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575870.593210:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575870.593212:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575870.593214:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575870.593215:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575870.593216:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575870.593218:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575870.848884:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311230701, at 4311230720 00000100:00000010:0.0:1788575870.848895:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf6c0. 00000020:00000001:0.0:1788575871.297029:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575871.297033:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575871.297036:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575871.616937:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575871.616947:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575871.616951:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575871.616959:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575871.616967:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575871.616973:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575871.617114:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575871.617117:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575871.617120:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575872.128933:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575872.128943:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575872.128946:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575872.128950:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575872.128953:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575872.128961:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16559 00000400:00000200:0.0:1788575872.128970:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575872.128976:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575872.128979:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575872.128982:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.128984:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575872.128986:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.128988:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575872.128992:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575872.128994:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575872.128996:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575872.128999:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575872.129001:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575872.129004:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575872.129007:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129009:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575872.129010:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129012:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575872.129031:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575872.129035:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129038:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575872.129041:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575872.129044:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575872.129047:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.129049:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575872.129051:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575872.129058:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575872.129062:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575872.129066:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575872.129067:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.129070:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575872.129073:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575872.129077:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575872.129084:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575872.129091:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575872.129099:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516876672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575872.129110:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575872.129114:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575872.129118:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516876672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575872.129136:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575872.129140:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575872.129144:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575872.129145:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575872.129146:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575872.129149:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575872.129152:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16559 00000400:00000200:0.0:1788575872.129156:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575872.129166:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575872.129169:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575872.129173:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129175:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575872.129178:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129180:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575872.129183:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575872.129185:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575872.129187:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575872.129190:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575872.129191:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575872.129194:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575872.129200:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129206:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575872.129220:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129224:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575872.129229:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575872.129236:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129242:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575872.129246:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575872.129252:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575872.129258:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.129263:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575872.129275:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575872.129280:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.129281:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575872.129286:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575872.129288:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575872.129294:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575872.129297:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.129300:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575872.129308:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575872.129311:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575872.129318:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575872.129322:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575872.129332:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516876800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575872.129352:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575872.129357:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575872.129367:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516876800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575872.129384:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575872.129388:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575872.129396:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575872.129398:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575872.129401:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575872.129406:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575872.129411:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16559 00000400:00000200:0.0:1788575872.129419:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575872.129428:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575872.129434:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575872.129441:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129446:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575872.129451:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129456:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575872.129461:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575872.129464:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575872.129468:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575872.129477:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575872.129480:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575872.129483:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575872.129496:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129499:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575872.129503:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129506:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575872.129516:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575872.129522:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129526:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575872.129530:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575872.129543:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575872.129548:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.129551:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575872.129556:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575872.129560:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.129563:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575872.129569:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575872.129573:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575872.129577:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575872.129579:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.129582:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575872.129583:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575872.129587:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575872.129593:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575872.129599:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575872.129617:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516876928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575872.129630:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575872.129633:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575872.129637:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516876928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575872.129644:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575872.129649:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575872.129658:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575872.129661:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575872.129664:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575872.129669:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575872.129673:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16559 00000400:00000200:0.0:1788575872.129678:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575872.129684:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575872.129687:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575872.129692:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129696:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575872.129699:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129701:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575872.129704:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575872.129706:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575872.129707:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575872.129710:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575872.129713:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575872.129717:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575872.129723:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129728:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575872.129734:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129739:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575872.129745:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575872.129763:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129769:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575872.129774:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575872.129778:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575872.129782:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.129786:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575872.129789:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575872.129798:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.129801:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575872.129835:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575872.129841:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575872.129846:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575872.129849:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.129854:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575872.129857:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575872.129862:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575872.129868:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575872.129873:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575872.129880:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516877056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575872.129894:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575872.129898:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575872.129902:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516877056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575872.129918:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575872.129924:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575872.129929:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575872.129933:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575872.129934:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575872.129939:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575872.129944:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16559 00000400:00000200:0.0:1788575872.129952:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575872.129958:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575872.129975:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575872.129981:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129985:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575872.129991:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.129999:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575872.130003:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575872.130005:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575872.130008:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575872.130011:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575872.130014:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575872.130017:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575872.130022:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.130025:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575872.130028:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575872.130033:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575872.130038:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575872.130042:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575872.130047:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575872.130051:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575872.130056:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575872.130063:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.130066:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575872.130071:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575872.130077:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.130082:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575872.130090:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575872.130096:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575872.130103:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575872.130106:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575872.130110:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575872.130113:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575872.130117:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575872.130125:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575872.130133:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575872.130144:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516877184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575872.130183:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575872.130187:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575872.130196:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516877184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575872.130220:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575872.130228:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16569) 00000100:00000001:0.0:1788575872.640947:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575872.640957:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575872.640962:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575872.640974:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575872.640982:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575872.640988:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575872.640995:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575872.640999:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575872.641008:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575873.347751:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575873.347756:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575873.347759:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575873.664930:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575873.664940:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575873.664946:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575873.664958:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575873.664965:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575873.664970:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575873.664983:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575873.664989:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575873.664998:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575874.369077:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575874.369081:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575874.369084:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788575875.393122:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575875.393127:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575875.393130:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575875.968978:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311235821, at 4311235840 00000100:00000010:0.0:1788575875.969007:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf5a0. 00000020:00000001:0.0:1788575876.417679:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575876.417683:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575876.417686:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575876.736969:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575876.736974:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575876.736977:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575876.736980:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575876.736985:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575876.736994:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575876.737002:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575876.737007:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575876.737016:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575877.248924:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575877.248936:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575877.248939:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575877.248943:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575877.248949:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575877.248956:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16564 00000400:00000200:0.0:1788575877.248967:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575877.248978:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575877.248986:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575877.249001:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249010:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575877.249015:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249017:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575877.249021:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575877.249023:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575877.249025:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575877.249028:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575877.249030:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575877.249033:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575877.249036:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249056:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575877.249059:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249061:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575877.249064:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575877.249067:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249070:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575877.249073:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575877.249075:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575877.249078:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.249080:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575877.249082:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575877.249089:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575877.249092:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575877.249096:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575877.249097:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.249101:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575877.249103:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575877.249107:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575877.249114:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575877.249120:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575877.249128:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516877312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575877.249138:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575877.249142:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575877.249146:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516877312/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575877.249169:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575877.249173:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575877.249176:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575877.249178:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575877.249180:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575877.249182:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575877.249193:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16564 00000400:00000200:0.0:1788575877.249197:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575877.249202:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575877.249206:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575877.249212:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249215:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575877.249220:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249224:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575877.249229:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575877.249231:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575877.249234:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575877.249238:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575877.249241:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575877.249246:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575877.249251:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249256:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575877.249262:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249267:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575877.249271:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575877.249275:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249279:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575877.249282:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575877.249291:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575877.249294:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.249300:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575877.249304:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575877.249309:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.249313:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575877.249319:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575877.249324:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575877.249328:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575877.249335:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.249340:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575877.249343:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575877.249347:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575877.249354:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575877.249359:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575877.249369:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516877440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575877.249382:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575877.249387:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575877.249394:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516877440/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575877.249408:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575877.249412:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575877.249418:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575877.249423:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575877.249427:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575877.249444:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575877.249451:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16564 00000400:00000200:0.0:1788575877.249471:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575877.249477:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575877.249481:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575877.249488:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249491:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575877.249495:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249500:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575877.249505:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575877.249508:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575877.249513:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575877.249518:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575877.249522:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575877.249533:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575877.249541:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249545:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575877.249551:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249556:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575877.249561:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575877.249566:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249571:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575877.249575:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575877.249579:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575877.249584:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.249590:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575877.249594:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575877.249598:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.249603:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575877.249608:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575877.249613:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575877.249617:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575877.249620:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.249622:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575877.249624:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575877.249626:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575877.249634:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575877.249642:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575877.249651:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516877568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575877.249666:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575877.249671:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575877.249680:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516877568/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575877.249693:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575877.249705:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575877.249713:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575877.249717:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575877.249721:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575877.249728:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575877.249736:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16564 00000400:00000200:0.0:1788575877.249752:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575877.249760:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575877.249765:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575877.249771:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249776:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575877.249782:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249788:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575877.249794:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575877.249798:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575877.249803:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575877.249831:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575877.249837:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575877.249841:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575877.249848:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249852:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575877.249857:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249861:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575877.249866:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575877.249872:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575877.249878:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575877.249882:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575877.249887:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575877.249892:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.249898:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575877.249907:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575877.249911:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.249915:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575877.249921:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575877.249926:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575877.249931:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575877.249934:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.249938:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575877.249942:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575877.249945:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575877.249952:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575877.249957:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575877.249966:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516877696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575877.249974:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575877.249976:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575877.249980:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516877696/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575877.249988:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575877.249993:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575877.249999:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575877.250003:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575877.250006:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575877.250010:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575877.250017:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16564 00000400:00000200:0.0:1788575877.250024:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575877.250031:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575877.250035:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575877.250042:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.250046:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575877.250052:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.250062:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575877.250067:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575877.250070:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575877.250075:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575877.250078:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575877.250082:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575877.250088:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575877.250094:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.250098:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575877.250103:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575877.250108:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575877.250113:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575877.250118:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575877.250122:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575877.250126:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575877.250130:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575877.250134:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.250138:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575877.250142:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575877.250146:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.250149:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575877.250156:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575877.250161:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575877.250165:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575877.250167:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575877.250171:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575877.250174:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575877.250178:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575877.250184:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575877.250191:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575877.250200:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516877824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575877.250238:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575877.250243:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575877.250251:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516877824/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575877.250266:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575877.250272:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16574) 00000100:00000001:0.0:1788575877.760989:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575877.760993:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575877.760994:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575877.760997:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575877.760999:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575877.761001:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575877.763399:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575877.763401:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575877.763403:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575878.465121:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575878.465126:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575878.465129:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575878.786159:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575878.786166:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575878.786170:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575878.786175:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575878.786182:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575878.786186:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575878.786191:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575878.786193:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575878.786196:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575879.489046:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575879.489053:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575879.489058:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575879.808967:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575879.808972:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575879.808977:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575879.808996:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575879.809021:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575879.809024:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575879.809026:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575879.809027:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575879.809037:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575880.513012:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575880.513016:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575880.513020:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575880.832974:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575880.832986:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575880.832990:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575880.833000:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575880.833011:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575880.833019:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575880.833054:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575880.833059:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575880.833063:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575881.088928:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311240941, at 4311240960 00000100:00000010:0.0:1788575881.088953:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfae0. 00000020:00000001:0.0:1788575881.537034:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575881.537038:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575881.537041:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575881.856913:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575881.856919:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575881.856922:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575881.856930:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575881.856936:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575881.856941:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575881.856947:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575881.856951:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575881.856957:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575881.857099:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575881.857105:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575881.857108:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575881.857111:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575881.857115:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575881.857118:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575881.857136:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575881.857137:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575881.857139:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575882.368922:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575882.368935:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575882.368938:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575882.368942:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575882.368947:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575882.368954:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16569 00000400:00000200:0.0:1788575882.368963:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575882.368969:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575882.368972:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575882.368975:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.368977:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575882.368980:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.368982:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575882.368986:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575882.368988:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575882.368990:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575882.368993:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575882.368995:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575882.368997:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575882.369000:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369002:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575882.369004:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369006:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575882.369008:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575882.369011:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369013:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575882.369016:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575882.369019:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575882.369037:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.369040:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575882.369042:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575882.369047:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575882.369058:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575882.369063:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575882.369065:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.369068:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575882.369071:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575882.369075:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575882.369082:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575882.369089:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575882.369098:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516877952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575882.369107:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575882.369111:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575882.369115:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516877952/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575882.369134:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575882.369137:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575882.369140:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575882.369141:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575882.369143:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575882.369146:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575882.369149:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16569 00000400:00000200:0.0:1788575882.369154:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575882.369161:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575882.369165:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575882.369172:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369178:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575882.369182:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369191:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575882.369197:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575882.369201:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575882.369205:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575882.369209:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575882.369213:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575882.369218:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575882.369224:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369226:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575882.369229:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369232:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575882.369236:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575882.369242:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369247:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575882.369250:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575882.369256:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575882.369267:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.369271:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575882.369275:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575882.369279:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.369284:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575882.369290:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575882.369294:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575882.369299:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575882.369303:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.369307:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575882.369310:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575882.369313:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575882.369319:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575882.369334:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575882.369343:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516878080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575882.369381:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575882.369385:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575882.369394:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516878080/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575882.369410:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575882.369414:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575882.369421:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575882.369425:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575882.369428:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575882.369432:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575882.369439:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16569 00000400:00000200:0.0:1788575882.369445:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575882.369451:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575882.369456:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575882.369461:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369472:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575882.369476:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369478:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575882.369481:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575882.369484:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575882.369487:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575882.369491:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575882.369495:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575882.369499:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575882.369505:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369507:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575882.369509:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369511:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575882.369520:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575882.369527:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369532:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575882.369534:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575882.369540:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575882.369545:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.369548:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575882.369552:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575882.369557:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.369561:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575882.369567:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575882.369572:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575882.369576:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575882.369579:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.369582:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575882.369585:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575882.369588:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575882.369594:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575882.369600:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575882.369617:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516878208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575882.369628:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575882.369631:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575882.369637:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516878208/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575882.369647:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575882.369651:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575882.369656:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575882.369658:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575882.369661:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575882.369665:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575882.369675:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16569 00000400:00000200:0.0:1788575882.369690:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575882.369697:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575882.369702:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575882.369707:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369711:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575882.369716:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369720:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575882.369730:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575882.369733:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575882.369737:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575882.369741:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575882.369744:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575882.369749:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575882.369755:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369757:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575882.369758:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369760:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575882.369762:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575882.369768:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369774:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575882.369778:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575882.369781:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575882.369785:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.369789:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575882.369792:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575882.369796:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.369799:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575882.369804:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575882.369837:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575882.369843:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575882.369851:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.369855:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575882.369859:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575882.369863:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575882.369870:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575882.369876:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575882.369884:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516878336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575882.369894:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575882.369899:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575882.369907:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516878336/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575882.369920:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575882.369924:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575882.369932:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575882.369935:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575882.369937:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575882.369944:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575882.369947:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16569 00000400:00000200:0.0:1788575882.369951:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575882.369957:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575882.369961:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575882.369968:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369973:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575882.369977:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.369980:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575882.369982:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575882.369983:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575882.369986:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575882.369990:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575882.369995:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575882.370003:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575882.370007:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.370010:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575882.370014:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575882.370019:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575882.370022:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575882.370028:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575882.370032:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575882.370036:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575882.370039:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575882.370043:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.370047:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575882.370051:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575882.370054:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.370055:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575882.370058:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575882.370061:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575882.370066:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575882.370069:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575882.370073:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575882.370074:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575882.370076:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575882.370080:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575882.370085:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575882.370094:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516878464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575882.370103:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575882.370106:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575882.370114:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516878464/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575882.370129:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575882.370139:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16579) 00000100:00000001:0.0:1788575882.880934:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575882.880942:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575882.880946:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575882.880951:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575882.880955:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575882.880959:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575882.880961:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575882.880963:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575882.880966:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575883.585028:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575883.585031:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575883.585034:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575883.905119:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575883.905125:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575883.905127:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575883.905130:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575883.905133:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575883.905135:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575883.905137:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575883.905139:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575883.905141:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575884.609065:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575884.609070:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575884.609073:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575884.928917:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575884.928923:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575884.928928:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575884.928936:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575884.928943:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575884.928949:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575884.928976:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575884.928979:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575884.928983:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575885.633022:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575885.633026:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575885.633029:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575886.208921:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311246061, at 4311246080 00000100:00000010:0.0:1788575886.208949:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf780. 00000020:00000001:0.0:1788575886.656989:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575886.656993:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575886.656997:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575886.976948:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575886.976954:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575886.976959:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575886.976964:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575886.976970:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575886.976975:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575886.976988:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575886.976992:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575886.976996:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575887.488906:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575887.488916:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575887.488919:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575887.488923:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575887.488928:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575887.488935:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16574 00000400:00000200:0.0:1788575887.488943:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575887.488959:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575887.488963:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575887.488966:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.488970:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575887.488972:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.488975:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575887.488979:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575887.488980:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575887.488982:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575887.488985:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575887.488987:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575887.488990:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575887.488993:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489012:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575887.489014:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489016:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575887.489018:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575887.489021:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489024:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575887.489026:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575887.489029:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575887.489032:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.489034:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575887.489036:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575887.489044:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575887.489047:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575887.489051:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575887.489053:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.489055:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575887.489058:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575887.489062:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575887.489069:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575887.489075:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575887.489084:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516878592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575887.489094:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575887.489098:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575887.489102:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516878592/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575887.489121:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575887.489125:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575887.489128:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575887.489129:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575887.489131:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575887.489134:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575887.489143:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16574 00000400:00000200:0.0:1788575887.489149:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575887.489155:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575887.489159:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575887.489165:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489170:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575887.489181:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489186:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575887.489190:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575887.489193:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575887.489197:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575887.489200:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575887.489205:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575887.489210:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575887.489217:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489234:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575887.489239:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489242:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575887.489246:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575887.489251:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489255:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575887.489258:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575887.489263:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575887.489267:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.489270:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575887.489274:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575887.489278:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.489281:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575887.489288:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575887.489291:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575887.489295:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575887.489303:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.489307:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575887.489310:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575887.489312:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575887.489316:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575887.489322:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575887.489330:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516878720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575887.489341:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575887.489345:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575887.489353:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516878720/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575887.489365:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575887.489370:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575887.489374:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575887.489377:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575887.489381:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575887.489385:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575887.489391:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16574 00000400:00000200:0.0:1788575887.489399:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575887.489406:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575887.489411:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575887.489418:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489422:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575887.489429:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489433:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575887.489438:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575887.489440:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575887.489443:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575887.489446:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575887.489452:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575887.489463:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575887.489472:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489477:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575887.489484:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489488:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575887.489495:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575887.489502:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489508:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575887.489522:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575887.489528:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575887.489533:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.489539:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575887.489544:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575887.489549:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.489552:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575887.489557:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575887.489563:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575887.489569:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575887.489572:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.489576:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575887.489580:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575887.489584:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575887.489592:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575887.489599:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575887.489609:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516878848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575887.489635:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575887.489640:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575887.489648:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516878848/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575887.489670:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575887.489682:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575887.489690:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575887.489695:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575887.489699:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575887.489708:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575887.489715:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16574 00000400:00000200:0.0:1788575887.489725:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575887.489732:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575887.489736:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575887.489744:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489749:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575887.489755:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489760:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575887.489764:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575887.489766:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575887.489770:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575887.489774:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575887.489780:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575887.489784:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575887.489791:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489797:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575887.489803:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489835:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575887.489842:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575887.489848:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575887.489853:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575887.489858:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575887.489863:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575887.489867:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.489871:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575887.489881:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575887.489886:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.489889:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575887.489892:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575887.489897:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575887.489901:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575887.489905:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.489910:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575887.489913:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575887.489918:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575887.489925:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575887.489931:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575887.489939:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516878976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575887.489952:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575887.489955:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575887.489964:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516878976/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575887.489981:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575887.489986:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575887.489994:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575887.489998:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575887.490002:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575887.490008:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575887.490014:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16574 00000400:00000200:0.0:1788575887.490020:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575887.490026:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575887.490032:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575887.490039:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.490045:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575887.490051:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.490063:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575887.490069:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575887.490073:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575887.490078:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575887.490082:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575887.490087:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575887.490092:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575887.490100:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.490105:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575887.490110:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575887.490113:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575887.490119:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575887.490126:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575887.490131:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575887.490136:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575887.490140:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575887.490144:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.490149:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575887.490153:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575887.490158:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.490163:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575887.490169:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575887.490174:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575887.490178:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575887.490181:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575887.490184:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575887.490187:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575887.490190:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575887.490198:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575887.490205:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575887.490214:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516879104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575887.490265:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575887.490271:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575887.490280:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516879104/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575887.490303:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575887.490311:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16585) 00000100:00000001:0.0:1788575888.001147:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575888.001151:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575888.001153:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575888.001156:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575888.001158:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575888.001160:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575888.001255:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575888.001258:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575888.001261:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575888.707367:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575888.707372:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575888.707377:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575889.024931:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575889.024939:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575889.024941:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575889.024946:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575889.024950:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575889.024954:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575889.024957:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575889.024959:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575889.024962:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575889.729010:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575889.729013:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575889.729015:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575890.049407:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575890.049411:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575890.049413:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575890.049417:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575890.049434:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575890.049436:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575890.049439:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575890.049440:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575890.049442:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575890.753074:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575890.753078:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575890.753082:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575891.072928:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575891.072941:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575891.072945:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575891.072949:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575891.072954:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575891.072958:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575891.072965:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575891.072972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575891.072975:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575891.073016:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575891.073021:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575891.073024:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575891.073032:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575891.073035:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575891.073038:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575891.073047:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575891.073050:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575891.073057:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575891.328993:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311251181, at 4311251200 00000100:00000010:0.0:1788575891.329020:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf120. 00000020:00000001:0.0:1788575891.777037:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575891.777041:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575891.777044:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575892.096921:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575892.096926:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575892.096929:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575892.096936:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575892.096944:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575892.096950:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575892.096973:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575892.096976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575892.096979:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575892.608884:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575892.608896:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575892.608904:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575892.608907:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575892.608912:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575892.608919:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16580 00000400:00000200:0.0:1788575892.608928:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575892.608948:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575892.608952:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575892.608955:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.608958:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575892.608960:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.608962:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575892.608967:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575892.608968:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575892.608971:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575892.608974:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575892.608976:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575892.608979:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575892.608981:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.608983:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575892.608985:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.608987:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575892.608990:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575892.608993:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575892.608995:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575892.608997:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575892.609001:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575892.609020:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.609023:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575892.609026:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575892.609031:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575892.609034:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575892.609038:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575892.609040:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.609042:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575892.609045:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575892.609049:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575892.609056:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575892.609064:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575892.609072:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516879232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575892.609082:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575892.609086:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575892.609090:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516879232/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575892.609109:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575892.609112:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575892.609144:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575892.609145:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575892.609148:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575892.609153:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575892.609156:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16580 00000400:00000200:0.0:1788575892.609160:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575892.609166:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575892.609170:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575892.609175:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609181:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575892.609185:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609196:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575892.609201:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575892.609203:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575892.609206:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575892.609208:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575892.609210:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575892.609214:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575892.609220:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609224:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575892.609230:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609233:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575892.609237:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575892.609242:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609245:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575892.609249:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575892.609254:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575892.609256:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.609258:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575892.609264:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575892.609268:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.609272:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575892.609278:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575892.609282:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575892.609285:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575892.609288:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.609291:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575892.609292:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575892.609295:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575892.609303:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575892.609309:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575892.609318:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516879360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575892.609347:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575892.609351:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575892.609358:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516879360/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575892.609371:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575892.609375:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575892.609380:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575892.609383:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575892.609385:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575892.609389:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575892.609395:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16580 00000400:00000200:0.0:1788575892.609404:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575892.609411:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575892.609417:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575892.609421:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609425:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575892.609430:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609435:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575892.609437:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575892.609440:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575892.609443:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575892.609446:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575892.609449:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575892.609453:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575892.609457:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609460:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575892.609464:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609467:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575892.609478:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575892.609485:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609491:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575892.609495:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575892.609498:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575892.609502:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.609506:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575892.609510:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575892.609516:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.609519:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575892.609524:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575892.609529:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575892.609535:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575892.609538:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.609542:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575892.609545:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575892.609548:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575892.609554:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575892.609559:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575892.609569:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516879488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575892.609581:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575892.609585:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575892.609592:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516879488/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575892.609607:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575892.609613:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575892.609619:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575892.609622:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575892.609625:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575892.609630:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575892.609640:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16580 00000400:00000200:0.0:1788575892.609648:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575892.609652:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575892.609653:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575892.609656:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609658:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575892.609680:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609687:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575892.609692:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575892.609694:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575892.609695:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575892.609698:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575892.609700:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575892.609703:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575892.609709:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609711:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575892.609714:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609719:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575892.609724:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575892.609729:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575892.609733:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575892.609736:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575892.609741:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575892.609746:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.609748:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575892.609750:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575892.609754:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.609777:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575892.609783:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575892.609788:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575892.609794:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575892.609802:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.609804:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575892.609806:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575892.609871:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575892.609880:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575892.609886:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575892.609895:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516879616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575892.609921:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575892.609926:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575892.609941:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516879616/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575892.609955:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575892.609965:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575892.609972:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575892.609976:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575892.609980:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575892.609984:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575892.609997:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16580 00000400:00000200:0.0:1788575892.610006:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575892.610020:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575892.610024:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575892.610033:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.610042:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575892.610048:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.610054:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575892.610058:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575892.610059:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575892.610061:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575892.610069:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575892.610074:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575892.610085:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575892.610100:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.610106:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575892.610112:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575892.610132:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575892.610139:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575892.610147:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575892.610158:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575892.610164:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575892.610170:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575892.610176:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.610188:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575892.610193:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575892.610199:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.610210:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575892.610216:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575892.610221:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575892.610227:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575892.610237:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575892.610242:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575892.610245:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575892.610248:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575892.610256:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575892.610268:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575892.610278:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516879744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575892.610301:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575892.610305:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575892.610320:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516879744/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575892.610345:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575892.610358:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16590) 00000100:00000001:0.0:1788575893.121061:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575893.121067:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575893.121069:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575893.121071:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575893.121074:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575893.121075:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575893.121077:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575893.121079:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575893.121080:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575893.825081:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575893.825083:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575893.825085:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788575894.849016:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575894.849019:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575894.849021:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575895.169166:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575895.169171:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575895.169175:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575895.169180:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575895.169185:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575895.169189:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575895.169193:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575895.169195:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575895.169197:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575895.873019:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575895.873023:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575895.873026:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575896.448896:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311256301, at 4311256320 00000100:00000010:0.0:1788575896.448911:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfb40. 00000020:00000001:0.0:1788575896.897440:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575896.897443:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575896.897446:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575897.217152:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575897.217154:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575897.217155:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575897.217157:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575897.217165:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575897.217167:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575897.217628:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575897.217630:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575897.217632:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575897.728917:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575897.728926:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575897.728929:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575897.728932:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575897.728935:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575897.728942:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16585 00000400:00000200:0.0:1788575897.728950:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575897.728956:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575897.728959:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575897.728962:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.728965:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575897.728968:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.728970:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575897.728974:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575897.728976:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575897.728978:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575897.728981:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575897.728983:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575897.728986:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575897.728989:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.728991:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575897.728993:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.728994:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575897.728996:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575897.728998:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729000:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575897.729035:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575897.729038:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575897.729041:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729043:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575897.729045:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575897.729052:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575897.729057:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575897.729060:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575897.729062:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729065:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575897.729068:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575897.729072:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575897.729080:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575897.729087:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575897.729096:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516879872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575897.729102:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575897.729106:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575897.729110:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516879872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575897.729131:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575897.729134:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575897.729137:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575897.729138:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575897.729140:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575897.729142:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575897.729145:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16585 00000400:00000200:0.0:1788575897.729149:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575897.729152:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575897.729154:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575897.729157:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729159:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575897.729167:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729169:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575897.729171:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575897.729173:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575897.729174:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575897.729176:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575897.729179:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575897.729181:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575897.729184:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729186:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575897.729188:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729190:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575897.729192:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575897.729194:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729197:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575897.729198:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575897.729202:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575897.729204:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729205:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575897.729208:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575897.729210:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729211:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575897.729214:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575897.729217:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575897.729219:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575897.729220:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729222:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575897.729223:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575897.729225:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575897.729229:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575897.729232:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575897.729237:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516880000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575897.729250:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575897.729252:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575897.729255:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516880000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575897.729263:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575897.729265:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575897.729267:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575897.729268:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575897.729269:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575897.729271:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575897.729273:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16585 00000400:00000200:0.0:1788575897.729276:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575897.729279:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575897.729280:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575897.729283:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729285:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575897.729287:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729289:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575897.729292:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575897.729293:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575897.729295:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575897.729297:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575897.729299:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575897.729301:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575897.729304:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729306:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575897.729308:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729310:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575897.729327:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575897.729331:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729333:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575897.729334:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575897.729336:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575897.729338:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729339:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575897.729340:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575897.729345:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729349:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575897.729352:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575897.729355:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575897.729357:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575897.729360:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729361:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575897.729362:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575897.729364:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575897.729366:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575897.729370:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575897.729374:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516880128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575897.729378:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575897.729381:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575897.729387:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516880128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575897.729398:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575897.729401:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575897.729404:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575897.729408:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575897.729414:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575897.729416:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575897.729426:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16585 00000400:00000200:0.0:1788575897.729431:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575897.729434:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575897.729436:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575897.729439:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729444:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575897.729447:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729449:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575897.729453:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575897.729456:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575897.729458:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575897.729460:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575897.729462:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575897.729463:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575897.729464:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729465:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575897.729466:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729467:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575897.729469:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575897.729471:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729473:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575897.729475:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575897.729478:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575897.729483:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729488:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575897.729490:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575897.729495:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729496:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575897.729500:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575897.729503:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575897.729505:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575897.729512:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729516:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575897.729519:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575897.729523:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575897.729528:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575897.729532:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575897.729536:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516880256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575897.729541:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575897.729545:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575897.729549:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516880256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575897.729559:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575897.729561:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575897.729573:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575897.729574:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575897.729576:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575897.729579:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575897.729584:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16585 00000400:00000200:0.0:1788575897.729595:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575897.729600:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575897.729607:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575897.729611:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729621:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575897.729627:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729631:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575897.729635:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575897.729636:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575897.729638:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575897.729640:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575897.729642:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575897.729651:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575897.729657:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729660:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575897.729661:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729663:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575897.729672:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575897.729676:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575897.729679:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575897.729682:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575897.729686:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575897.729688:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729690:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575897.729692:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575897.729694:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729696:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575897.729699:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575897.729703:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575897.729705:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575897.729707:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575897.729708:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575897.729710:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575897.729712:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575897.729717:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575897.729721:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575897.729727:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516880384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575897.729747:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575897.729751:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575897.729760:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516880384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575897.729768:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575897.729779:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16595) 00000020:00000001:0.0:1788575898.946271:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575898.946276:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575898.946279:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575899.264936:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575899.264942:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575899.264947:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575899.264954:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575899.264959:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575899.264964:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575899.264969:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575899.264973:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575899.264976:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575899.969117:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575899.969121:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575899.969124:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575900.288914:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575900.288922:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575900.288924:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575900.288928:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575900.288933:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575900.288937:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575900.288940:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575900.288942:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575900.288947:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788575900.801078:0:2388:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1788575900.801126:0:2388:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788575900.801134:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575900.801138:0:2388:0:(router.c:1791:lnet_notify()) 192.168.202.28@tcp notifying 192.168.202.128@tcp: down 00000400:00000200:0.0:1788575900.801141:0:2388:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575900.801144:0:2388:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1788575900.801147:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575900.801150:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793900. 00000400:00000200:0.0:1788575900.801154:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575900.801157:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575900.801162:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcee0 00000400:00000010:0.0:1788575900.801176:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcee0. 00000100:00000001:0.0:1788575900.801181:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575900.801189:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c0700 x1875440516879872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575900.801213:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575900.801217:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c0700 x1875440516879872/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575922 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575900.801225:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575900.801226:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575900.801235:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575900.801238:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793000. 00000400:00000200:0.0:1788575900.801240:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575900.801244:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575900.801247:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fca18 00000400:00000010:0.0:1788575900.801248:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fca18. 00000100:00000001:0.0:1788575900.801251:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575900.801255:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c2300 x1875440516880000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575900.801260:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575900.801262:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c2300 x1875440516880000/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575922 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575900.801266:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575900.801267:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575900.801272:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575900.801273:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793b00. 00000400:00000200:0.0:1788575900.801275:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575900.801277:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575900.801280:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc330 00000400:00000010:0.0:1788575900.801281:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc330. 00000100:00000001:0.0:1788575900.801283:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575900.801289:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c1f80 x1875440516880128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575900.801321:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575900.801327:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c1f80 x1875440516880128/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575922 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575900.801334:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575900.801337:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575900.801346:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575900.801351:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793400. 00000400:00000200:0.0:1788575900.801355:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575900.801360:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575900.801363:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fcaa0 00000400:00000010:0.0:1788575900.801366:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fcaa0. 00000100:00000001:0.0:1788575900.801369:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575900.801376:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c2680 x1875440516880256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575900.801388:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575900.801395:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c2680 x1875440516880256/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575922 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575900.801428:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575900.801432:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788575900.801442:0:2388:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000010:0.0:1788575900.801450:0:2388:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4a0793d00. 00000400:00000200:0.0:1788575900.801454:0:2388:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788575900.801461:0:2388:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788575900.801466:0:2388:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc4a70fc110 00000400:00000010:0.0:1788575900.801469:0:2388:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc4a70fc110. 00000100:00000001:0.0:1788575900.801473:0:2388:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575900.801481:0:2388:0:(events.c:40:request_out_callback()) @@@ type 5, status -113 req@ffff8fc4b80c1c00 x1875440516880384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575900.801497:0:2388:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575900.801504:0:2388:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4b80c1c00 x1875440516880384/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575922 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575900.801524:0:2388:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575900.801528:0:2388:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788575900.993099:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575900.993103:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575900.993106:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575901.313019:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575901.313023:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575901.313033:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575901.313037:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575901.313039:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575901.313042:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575901.313044:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575901.313045:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575901.313047:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575901.568948:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311261422, at 4311261440 00000100:00000010:0.0:1788575901.568964:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cff00. 00000020:00000001:0.0:1788575902.017032:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575902.017036:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575902.017040:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575902.336956:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575902.336961:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575902.336964:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575902.336970:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575902.336975:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575902.336980:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.337098:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575902.337100:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.337113:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.852106:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575902.852110:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575902.852113:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575902.852120:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516880512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.852128:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575902.852145:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575902.852148:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575902.852151:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575902.852153:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575902.852156:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516880512:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575902.852160:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575902.852165:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575902.852167:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575902.852169:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575902.852173:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575902.852175:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575902.852181:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575902.852185:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575902.852192:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575902.852198:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516880512, portal 25 00000100:00000001:0.0:1788575902.852201:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575902.852202:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575902.852206:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516880512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.852213:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575902.852215:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575902.852218:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575902.852221:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516880512, offset 0 00000400:00000200:0.0:1788575902.852226:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575902.852233:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575902.852239:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575902.852244:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575902.852247:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575902.852251:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575902.852253:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575902.852256:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548531:548531:256:4294967295] 192.168.202.128@tcp LPNI seq info [548454:548454:8:4294967295] 00000400:00000200:0.0:1788575902.852260:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575902.852269:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575902.852281:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575902.852286:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575902.852293:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575902.852297:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788575902.852310:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575902.852312:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575902.852318:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575902.852322:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575902.852328:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575902.852331:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575902.852337:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575902.852341:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516880512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.854971:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575902.854986:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516880512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.854998:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.855001:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575902.855014:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575902.855020:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575902.855023:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311266544 00000100:00000010:0.0:1788575902.855031:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50840. 00000100:00000001:0.0:1788575902.855039:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575902.855046:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311266544 00000100:00000010:0.0:1788575902.855049:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50600. 00000100:00000001:0.0:1788575902.855051:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575902.855054:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575902.855057:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.855059:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575902.855064:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575902/real 1788575902] req@ffff8fc4901a7800 x1875440516880512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575902.855094:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575902.855095:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575902.855098:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575902.855102:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575902.855105:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575902.855109:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575902.855113:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516880512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575902.855122:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516880512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.855129:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575902.855131:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.855133:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575902.855135:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575902.855139:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516880512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.855145:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575902.855150:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516880512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.855157:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575902.855160:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.855162:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575902.855163:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.855166:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575902.855168:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575902.855171:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575902.855175:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575902.855178:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575902.855182:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516880512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575902.855197:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516880512:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575902.855202:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.855205:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575902.855208:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516880512/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.855214:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575902.855216:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575902.855219:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575902.855223:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575902.855225:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575902.855227:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575902.855229:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575902.855232:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575902.855236:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575902.855238:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575902.855239:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575902.855243:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575902.855244:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.855246:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.855254:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575902.855255:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.855258:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575902.855261:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575902.855262:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575902.855268:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516880640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.855275:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575902.855277:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575902.855280:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575902.855283:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575902.855286:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575902.855289:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516880640:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575902.855299:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575902.855302:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575902.855304:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575902.855307:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575902.855311:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575902.855313:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575902.855318:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575902.855321:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575902.855325:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575902.855330:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516880640, portal 10 00000100:00000001:0.0:1788575902.855332:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575902.855334:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575902.855341:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516880640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.855347:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575902.855349:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575902.855351:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575902.855354:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516880640, offset 0 00000400:00000200:0.0:1788575902.855360:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575902.855365:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575902.855372:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575902.855377:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575902.855380:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575902.855383:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575902.855386:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575902.855389:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548532:548532:256:4294967295] 192.168.202.128@tcp LPNI seq info [548455:548455:8:4294967295] 00000400:00000200:0.0:1788575902.855394:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575902.855401:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575902.855406:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575902.855411:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575902.855417:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575902.855425:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788575902.855437:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575902.855439:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575902.855444:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575902.855448:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575902.855453:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575902.855455:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575902.855457:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575902.855461:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516880640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.855468:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575902.855471:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516880640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.855477:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.855478:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575902.855481:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575902.855486:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575902.855489:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575902.855491:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311262741 00000100:00000001:0.0:1788575902.855494:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311266544 00000100:00000001:0.0:1788575902.855496:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311262741 00000100:00000001:0.0:1788575902.855498:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311266544 00000100:00000001:0.0:1788575902.855500:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575902.855502:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.855504:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575902.855507:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575902/real 1788575902] req@ffff8fc4901a5c00 x1875440516880640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575902.855514:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575902.855515:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575902.855522:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575902.855525:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575902.855526:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575902.855529:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575902.855532:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516880640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575902.855539:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516880640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.855545:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575902.855546:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.855548:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575902.855549:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575902.855552:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516880640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.855558:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575902.855562:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516880640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.855568:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575902.855570:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.855572:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575902.855573:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.855574:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575902.855577:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575902.855579:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575902.855582:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575902.855584:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575902.855587:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516880640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575902.855594:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516880640:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575902.855602:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.855604:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575902.855607:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516880640/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.855613:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575902.855615:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575902.855616:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575902.855619:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575902.855620:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575902.855622:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575902.855624:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575902.855626:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575902.855631:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575902.855632:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575902.855634:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575902.855639:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575902.855640:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.855642:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.855646:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575902.855647:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.855649:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575902.855651:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.856792:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575902.856796:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575902.856798:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575902.856803:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516880768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.856832:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575902.856833:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575902.856836:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575902.856838:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575902.856840:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575902.856844:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516880768:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575902.856847:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575902.856859:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575902.856861:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575902.856863:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575902.856866:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575902.856868:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575902.856872:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575902.856876:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575902.856881:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575902.856885:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516880768, portal 10 00000100:00000001:0.0:1788575902.856887:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575902.856889:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575902.856895:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516880768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.856902:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575902.856914:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575902.856916:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575902.856919:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516880768, offset 0 00000400:00000200:0.0:1788575902.856924:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575902.856929:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575902.856934:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575902.856938:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575902.856941:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575902.856944:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575902.856947:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575902.856949:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548533:548533:256:4294967295] 192.168.202.128@tcp LPNI seq info [548456:548456:8:4294967295] 00000400:00000200:0.0:1788575902.856953:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575902.856960:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575902.856965:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575902.856970:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575902.856975:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575902.856978:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788575902.856990:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575902.856997:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575902.857001:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575902.857004:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575902.857009:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575902.857011:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575902.857014:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575902.857018:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516880768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857025:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575902.857028:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516880768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857034:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.857035:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575902.857039:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575902.857042:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575902.857046:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575902.857048:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311262753 00000100:00000001:0.0:1788575902.857050:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311266544 00000100:00000001:0.0:1788575902.857052:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311262753 00000100:00000001:0.0:1788575902.857054:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311266544 00000100:00000001:0.0:1788575902.857056:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575902.857058:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.857060:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575902.857064:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575902/real 1788575902] req@ffff8fc4901a5180 x1875440516880768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575902.857070:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575902.857072:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575902.857073:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575902.857075:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575902.857076:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575902.857084:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575902.857088:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516880768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575902.857094:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516880768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857100:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575902.857101:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.857103:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575902.857105:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575902.857109:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516880768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857114:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575902.857118:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516880768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857124:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575902.857127:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.857128:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575902.857129:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.857131:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575902.857133:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575902.857136:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575902.857139:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575902.857140:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575902.857144:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516880768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575902.857151:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516880768:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575902.857155:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575902.857163:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516880896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857169:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575902.857171:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575902.857173:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575902.857174:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575902.857176:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575902.857178:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516880896:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575902.857181:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575902.857183:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575902.857184:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575902.857186:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575902.857189:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575902.857190:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575902.857194:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575902.857196:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575902.857200:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575902.857203:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516880896, portal 4 00000100:00000001:0.0:1788575902.857205:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575902.857206:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575902.857211:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516880896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857219:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575902.857221:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575902.857223:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575902.857225:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516880896, offset 0 00000400:00000200:0.0:1788575902.857229:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575902.857233:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575902.857236:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575902.857239:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575902.857241:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575902.857244:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575902.857246:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575902.857256:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548534:548534:256:4294967295] 192.168.202.128@tcp LPNI seq info [548457:548457:8:4294967295] 00000400:00000200:0.0:1788575902.857260:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575902.857266:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575902.857271:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575902.857274:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575902.857278:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575902.857281:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788575902.857291:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575902.857293:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575902.857296:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575902.857299:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575902.857302:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575902.857304:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575902.857307:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575902.857310:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516880896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857316:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575902.857318:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516880896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857324:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.857325:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575902.857328:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575902.857331:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575902.857333:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575902.857335:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311262893 00000100:00000001:0.0:1788575902.857337:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311266544 00000100:00000001:0.0:1788575902.857339:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311262893 00000100:00000001:0.0:1788575902.857341:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311266544 00000100:00000001:0.0:1788575902.857342:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575902.857349:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.857351:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575902.857355:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575902/real 1788575902] req@ffff8fc4901a7100 x1875440516880896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575902.857362:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575902.857363:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575902.857365:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575902.857368:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575902.857369:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575902.857371:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575902.857374:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516880896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575902.857381:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516880896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857387:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575902.857389:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.857391:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575902.857392:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575902.857395:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516880896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857401:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575902.857405:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516880896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857411:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575902.857413:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.857415:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575902.857416:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.857418:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575902.857419:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575902.857421:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575902.857431:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575902.857433:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575902.857437:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516880896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575902.857445:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516880896:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575902.857449:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575902.857452:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516881024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857458:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575902.857459:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575902.857461:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575902.857463:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575902.857465:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575902.857468:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516881024:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575902.857470:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575902.857472:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575902.857473:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575902.857475:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575902.857479:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575902.857481:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575902.857484:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575902.857487:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575902.857490:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575902.857493:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516881024, portal 4 00000100:00000001:0.0:1788575902.857495:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575902.857496:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575902.857502:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516881024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857507:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575902.857515:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575902.857517:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575902.857519:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516881024, offset 0 00000400:00000200:0.0:1788575902.857523:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575902.857527:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575902.857530:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575902.857532:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575902.857534:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575902.857536:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575902.857538:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575902.857540:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548535:548535:256:4294967295] 192.168.202.128@tcp LPNI seq info [548458:548458:8:4294967295] 00000400:00000200:0.0:1788575902.857543:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575902.857549:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575902.857552:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575902.857555:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575902.857558:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575902.857560:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788575902.857568:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575902.857570:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575902.857573:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575902.857576:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575902.857578:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575902.857580:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575902.857582:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575902.857585:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516881024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857591:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575902.857594:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516881024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575927 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857598:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.857600:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575902.857607:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575902.857610:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575902.857613:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575902.857614:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311262895 00000100:00000001:0.0:1788575902.857616:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311266544 00000100:00000001:0.0:1788575902.857618:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311262895 00000100:00000001:0.0:1788575902.857619:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311266544 00000100:00000001:0.0:1788575902.857621:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575902.857623:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.857624:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575902.857627:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575902/real 1788575902] req@ffff8fc4901a4a80 x1875440516881024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575902.857634:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575902.857635:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575902.857637:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575902.857639:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575902.857641:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575902.857642:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575902.857645:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516881024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575902.857653:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516881024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857658:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575902.857660:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.857661:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575902.857663:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575902.857666:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516881024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.857672:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575902.857676:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516881024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.858794:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575902.858798:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.858800:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575902.858802:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.858804:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575902.858806:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575902.858822:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575902.858829:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575902.858831:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575902.858837:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516881024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575902.858847:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516881024:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575902.858851:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.858854:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575902.858857:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516880768/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.858864:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575902.858866:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575902.858868:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575902.858873:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575902.858875:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575902.858876:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575902.858878:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575902.858880:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575902.858884:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575902.858886:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575902.858888:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575902.858893:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575902.858894:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.858913:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575902.858918:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516880896/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.858924:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575902.858926:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575902.858927:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575902.858930:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575902.858932:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575902.858933:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575902.858935:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575902.858937:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575902.858940:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575902.858941:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575902.858942:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575902.858945:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575902.858946:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.858948:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575902.858951:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516881024/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575927 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575902.858957:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575902.858958:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575902.858959:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9fc00. 02000000:00000001:0.0:1788575902.858962:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575902.858963:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575902.858964:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575902.858966:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575902.858967:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575902.858969:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575902.858970:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575902.858971:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575902.858975:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575902.858976:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.858978:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575902.858985:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575902.858987:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575902.858990:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575902.858999:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575903.361206:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575903.361211:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575903.361213:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575903.361216:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575903.361219:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575903.361221:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575903.361223:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575903.361224:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575903.361226:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575904.065173:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575904.065177:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575904.065180:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575904.384967:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575904.384975:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575904.384977:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575904.384982:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575904.384986:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575904.384990:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575904.384994:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575904.384996:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575904.384999:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575905.089380:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575905.089382:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575905.089384:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575905.408972:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575905.408978:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575905.408981:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575905.408985:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575905.408989:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575905.408991:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575905.411910:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575905.411914:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575905.411918:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575906.113046:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575906.113050:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575906.113053:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575906.689039:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311266544, at 4311266560 00000100:00000010:0.0:1788575906.689061:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50840. 00000020:00000001:0.0:1788575907.137115:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575907.137118:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575907.137121:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575907.456976:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575907.456980:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575907.456982:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575907.456986:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575907.456992:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575907.457004:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.457011:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575907.457014:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.457016:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.970382:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575907.970386:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575907.970389:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575907.970395:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821c00 x1875440516881152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.970414:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575907.970418:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575907.970424:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575907.970429:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575907.970433:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575907.970441:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516881152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575907.970450:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575907.970456:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575907.970465:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575907.970468:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575907.970474:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575907.970476:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575907.970484:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575907.970490:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575907.970499:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575907.970509:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516881152, portal 25 00000100:00000001:0.0:1788575907.970513:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575907.970523:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410944 : -123400106140672 : ffff8fc4ac821c00) 00000100:00000040:0.0:1788575907.970534:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821c00 x1875440516881152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.970543:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575907.970549:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575907.970554:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575907.970560:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516881152, offset 0 00000400:00000200:0.0:1788575907.970569:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575907.970579:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575907.970587:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575907.970592:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575907.970596:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575907.970602:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575907.970616:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575907.970625:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548536:548536:256:4294967295] 192.168.202.128@tcp LPNI seq info [548459:548459:8:4294967295] 00000400:00000200:0.0:1788575907.970636:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575907.970653:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575907.970666:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575907.970674:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575907.970684:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575907.970690:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575907.970721:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575907.970726:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575907.970733:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575907.970739:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575907.970746:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575907.970751:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575907.970758:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575907.970763:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821c00 x1875440516881152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.970775:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575907.970787:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821c00 x1875440516881152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.970853:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.970857:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575907.970866:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575907.970874:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575907.970879:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311271663 00000100:00000010:0.0:1788575907.970885:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50840. 00000100:00000001:0.0:1788575907.970894:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575907.970902:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311271663 00000100:00000010:0.0:1788575907.970908:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50060. 00000100:00000001:0.0:1788575907.970914:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575907.970921:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575907.970928:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.970932:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575907.970940:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575907/real 1788575907] req@ffff8fc4ac821c00 x1875440516881152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575907.970973:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575907.970977:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575907.970981:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575907.970988:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575907.970992:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575907.971001:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575907.971008:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821c00 x1875440516881152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575907.971026:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821c00 x1875440516881152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971036:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575907.971039:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.971044:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575907.971046:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575907.971051:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821c00 x1875440516881152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971066:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575907.971074:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821c00 x1875440516881152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971092:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575907.971098:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.971102:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575907.971107:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.971113:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575907.971118:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575907.971123:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575907.971130:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575907.971134:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575907.971139:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821c00 x1875440516881152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575907.971151:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516881152:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575907.971158:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.971160:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575907.971165:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821c00 x1875440516881152/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971185:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575907.971189:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575907.971193:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575907.971197:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575907.971200:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575907.971204:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575907.971206:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575907.971211:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b000. 02000000:00000001:0.0:1788575907.971215:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575907.971223:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575907.971226:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821c00. 00000100:00000001:0.0:1788575907.971232:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575907.971235:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.971237:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.971245:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575907.971247:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.971252:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575907.971254:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575907.971257:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575907.971262:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac821880 x1875440516881280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971272:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575907.971274:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575907.971276:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575907.971279:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575907.971282:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575907.971287:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516881280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575907.971292:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575907.971293:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575907.971295:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575907.971305:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575907.971313:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575907.971317:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575907.971324:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575907.971329:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575907.971335:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575907.971341:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516881280, portal 10 00000100:00000001:0.0:1788575907.971346:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575907.971348:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603410048 : -123400106141568 : ffff8fc4ac821880) 00000100:00000040:0.0:1788575907.971357:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac821880 x1875440516881280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971363:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575907.971366:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575907.971375:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575907.971379:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516881280, offset 0 00000400:00000200:0.0:1788575907.971384:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575907.971388:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575907.971392:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575907.971395:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575907.971398:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575907.971400:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575907.971402:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575907.971405:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548537:548537:256:4294967295] 192.168.202.128@tcp LPNI seq info [548460:548460:8:4294967295] 00000400:00000200:0.0:1788575907.971409:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575907.971415:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575907.971420:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575907.971423:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575907.971427:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575907.971429:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575907.971440:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575907.971442:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575907.971445:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575907.971448:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575907.971451:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575907.971453:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575907.971456:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575907.971460:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac821880 x1875440516881280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971467:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575907.971470:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac821880 x1875440516881280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971476:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.971477:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575907.971486:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575907.971489:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575907.971493:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575907.971495:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311267860 00000100:00000001:0.0:1788575907.971497:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311271663 00000100:00000001:0.0:1788575907.971499:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311267860 00000100:00000001:0.0:1788575907.971501:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311271663 00000100:00000001:0.0:1788575907.971503:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575907.971505:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.971506:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575907.971511:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575907/real 1788575907] req@ffff8fc4ac821880 x1875440516881280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575907.971517:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575907.971518:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575907.971520:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575907.971522:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575907.971524:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575907.971526:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575907.971528:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac821880 x1875440516881280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575907.971536:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac821880 x1875440516881280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971542:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575907.971543:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.971545:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575907.971546:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575907.971550:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac821880 x1875440516881280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971555:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575907.971560:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac821880 x1875440516881280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971572:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575907.971575:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.971577:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575907.971578:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.971580:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575907.971582:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575907.971584:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575907.971588:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575907.971590:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575907.971593:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac821880 x1875440516881280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575907.971600:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac821880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516881280:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575907.971604:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575907.971607:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820a80 x1875440516881408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971613:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575907.971614:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575907.971616:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575907.971617:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575907.971620:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575907.971623:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516881408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575907.971626:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575907.971628:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575907.971630:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575907.971631:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575907.971634:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575907.971636:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575907.971644:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575907.971647:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575907.971651:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575907.971654:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516881408, portal 10 00000100:00000001:0.0:1788575907.971656:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575907.971657:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603406464 : -123400106145152 : ffff8fc4ac820a80) 00000100:00000040:0.0:1788575907.971663:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820a80 x1875440516881408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971669:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575907.971671:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575907.971673:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575907.971675:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516881408, offset 0 00000400:00000200:0.0:1788575907.971679:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575907.971683:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575907.971686:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575907.971688:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575907.971691:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575907.971692:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575907.971694:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575907.971697:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548538:548538:256:4294967295] 192.168.202.128@tcp LPNI seq info [548461:548461:8:4294967295] 00000400:00000200:0.0:1788575907.971701:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575907.971707:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575907.971711:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575907.971714:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575907.971718:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575907.971720:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575907.971736:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575907.971738:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575907.971741:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575907.971744:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575907.971748:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575907.971753:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575907.971756:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575907.971760:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820a80 x1875440516881408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971768:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575907.971771:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820a80 x1875440516881408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971777:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.971779:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575907.971782:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575907.971785:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575907.971788:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575907.971790:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311267867 00000100:00000001:0.0:1788575907.971793:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311271663 00000100:00000001:0.0:1788575907.971795:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311267867 00000100:00000001:0.0:1788575907.971797:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311271663 00000100:00000001:0.0:1788575907.971831:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575907.971834:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.971835:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575907.971839:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575907/real 1788575907] req@ffff8fc4ac820a80 x1875440516881408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575907.971847:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575907.971849:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575907.971850:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575907.971853:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575907.971854:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575907.971857:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575907.971860:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820a80 x1875440516881408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575907.971867:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820a80 x1875440516881408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971879:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575907.971881:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.971883:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575907.971884:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575907.971888:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820a80 x1875440516881408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971904:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575907.971909:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820a80 x1875440516881408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971915:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575907.971917:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.971919:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575907.971920:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.971922:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575907.971924:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575907.971927:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575907.971930:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575907.971932:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575907.971936:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820a80 x1875440516881408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575907.971943:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516881408:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575907.971947:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575907.971950:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac823800 x1875440516881536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.971956:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575907.971958:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575907.971966:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575907.971967:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575907.971969:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575907.971973:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516881536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575907.971977:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575907.971984:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575907.971986:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575907.971988:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575907.971991:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575907.971993:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575907.971997:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575907.971999:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575907.972003:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575907.972006:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516881536, portal 4 00000100:00000001:0.0:1788575907.972008:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575907.972009:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603418112 : -123400106133504 : ffff8fc4ac823800) 00000100:00000040:0.0:1788575907.972015:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac823800 x1875440516881536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972021:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575907.972023:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575907.972025:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575907.972027:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516881536, offset 0 00000400:00000200:0.0:1788575907.972031:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575907.972035:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575907.972039:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575907.972041:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575907.972044:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575907.972046:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575907.972048:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575907.972051:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548539:548539:256:4294967295] 192.168.202.128@tcp LPNI seq info [548462:548462:8:4294967295] 00000400:00000200:0.0:1788575907.972055:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575907.972061:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575907.972071:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575907.972074:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575907.972078:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575907.972080:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575907.972090:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575907.972092:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575907.972095:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575907.972098:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575907.972101:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575907.972103:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575907.972106:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575907.972109:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac823800 x1875440516881536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972117:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575907.972120:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac823800 x1875440516881536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972126:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.972128:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575907.972131:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575907.972134:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575907.972137:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575907.972138:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311268008 00000100:00000001:0.0:1788575907.972141:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311271663 00000100:00000001:0.0:1788575907.972143:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311268008 00000100:00000001:0.0:1788575907.972145:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311271663 00000100:00000001:0.0:1788575907.972147:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575907.972149:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.972151:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575907.972154:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575907/real 1788575907] req@ffff8fc4ac823800 x1875440516881536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575907.972168:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575907.972169:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575907.972171:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575907.972173:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575907.972175:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575907.972177:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575907.972181:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac823800 x1875440516881536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575907.972189:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac823800 x1875440516881536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972195:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575907.972197:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972199:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575907.972200:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575907.972203:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac823800 x1875440516881536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972209:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575907.972213:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac823800 x1875440516881536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972219:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575907.972221:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972223:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575907.972224:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972226:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575907.972228:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575907.972230:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575907.972233:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575907.972235:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575907.972244:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac823800 x1875440516881536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575907.972252:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac823800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516881536:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575907.972255:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972258:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575907.972261:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac821880 x1875440516881280/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972267:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575907.972269:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575907.972271:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575907.972274:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575907.972275:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575907.972276:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575907.972278:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575907.972280:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629000. 02000000:00000001:0.0:1788575907.972283:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575907.972284:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575907.972286:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac821880. 00000100:00000001:0.0:1788575907.972288:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575907.972289:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972291:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575907.972294:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820a80 x1875440516881408/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972300:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575907.972302:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575907.972303:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575907.972306:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575907.972307:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575907.972308:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575907.972310:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575907.972311:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862a400. 02000000:00000001:0.0:1788575907.972314:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575907.972322:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575907.972323:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820a80. 00000100:00000001:0.0:1788575907.972326:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575907.972327:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972329:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575907.972332:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac823800 x1875440516881536/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972338:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575907.972339:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575907.972341:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575907.972344:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575907.972345:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575907.972346:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575907.972348:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575907.972350:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b8629400. 02000000:00000001:0.0:1788575907.972352:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575907.972353:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575907.972354:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac823800. 00000100:00000001:0.0:1788575907.972357:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575907.972358:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972359:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972364:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575907.972365:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.972367:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575907.972368:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575907.972370:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575907.972373:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4ac820e00 x1875440516881664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972380:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575907.972381:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575907.972384:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575907.972385:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575907.972387:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575907.972390:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516881664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575907.972393:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575907.972400:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575907.972401:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575907.972403:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575907.972406:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575907.972408:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575907.972411:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575907.972414:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575907.972417:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575907.972421:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516881664, portal 4 00000100:00000001:0.0:1788575907.972423:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575907.972424:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673603407360 : -123400106144256 : ffff8fc4ac820e00) 00000100:00000040:0.0:1788575907.972430:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4ac820e00 x1875440516881664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972436:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575907.972438:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575907.972440:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575907.972442:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516881664, offset 0 00000400:00000200:0.0:1788575907.972446:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575907.972449:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575907.972453:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575907.972456:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575907.972458:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575907.972460:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575907.972463:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575907.972466:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548540:548540:256:4294967295] 192.168.202.128@tcp LPNI seq info [548463:548463:8:4294967295] 00000400:00000200:0.0:1788575907.972470:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575907.972475:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575907.972480:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575907.972483:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575907.972488:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575907.972490:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1788575907.972500:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575907.972507:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575907.972509:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575907.972513:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575907.972516:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575907.972518:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575907.972520:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575907.972524:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4ac820e00 x1875440516881664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972531:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575907.972534:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4ac820e00 x1875440516881664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575932 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972540:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.972541:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575907.972544:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575907.972547:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575907.972550:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575907.972552:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311268010 00000100:00000001:0.0:1788575907.972554:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311271663 00000100:00000001:0.0:1788575907.972557:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311268010 00000100:00000001:0.0:1788575907.972559:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311271663 00000100:00000001:0.0:1788575907.972561:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575907.972563:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.972565:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575907.972568:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575907/real 1788575907] req@ffff8fc4ac820e00 x1875440516881664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575907.972575:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575907.972577:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575907.972578:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575907.972580:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575907.972582:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575907.972589:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575907.972593:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4ac820e00 x1875440516881664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575907.972600:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4ac820e00 x1875440516881664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972606:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575907.972608:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972609:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575907.972611:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575907.972614:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4ac820e00 x1875440516881664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972620:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575907.972623:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4ac820e00 x1875440516881664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972630:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575907.972632:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972634:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575907.972635:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972637:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575907.972638:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575907.972641:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575907.972644:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575907.972646:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575907.972650:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4ac820e00 x1875440516881664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575907.972658:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4ac820e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516881664:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575907.972661:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972663:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575907.972670:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4ac820e00 x1875440516881664/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575932 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575907.972676:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575907.972677:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575907.972679:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575907.972681:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575907.972682:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575907.972684:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575907.972685:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575907.972687:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4b862b800. 02000000:00000001:0.0:1788575907.972689:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575907.972690:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575907.972692:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4ac820e00. 00000100:00000001:0.0:1788575907.972694:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575907.972695:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972697:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575907.972700:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575907.972701:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575907.972702:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575907.972704:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575908.482890:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575908.482895:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575908.482896:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575908.482900:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575908.482903:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575908.482906:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575908.482987:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575908.482991:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575908.482993:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575909.185051:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575909.185054:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575909.185056:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575909.505031:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575909.505035:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575909.505038:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575909.505043:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575909.505047:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575909.505062:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575909.507424:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575909.507427:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575909.507430:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575910.209099:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575910.209103:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575910.209106:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575910.528969:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575910.528978:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575910.528981:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575910.528986:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575910.528990:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575910.528994:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575910.528997:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575910.529001:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575910.529004:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575910.529040:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575910.529044:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575910.529047:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575910.529053:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575910.529060:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575910.529065:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575910.529070:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575910.529074:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575910.529078:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575911.233008:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575911.233010:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575911.233013:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575911.808914:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311271663, at 4311271680 00000100:00000010:0.0:1788575911.808931:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50840. 00000020:00000001:0.0:1788575912.257005:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575912.257009:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575912.257012:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575912.577075:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575912.577078:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575912.577080:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575912.577083:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575912.577093:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575912.577096:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575912.577098:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575912.577099:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575912.577102:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575913.088921:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575913.088931:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575913.088934:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575913.088937:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575913.088941:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575913.088948:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16600 00000400:00000200:0.0:1788575913.088956:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575913.088962:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575913.088965:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575913.088968:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.088971:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575913.088973:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.088976:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575913.088980:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575913.088982:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575913.088985:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575913.088988:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575913.088990:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575913.088993:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575913.088996:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.088998:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575913.088999:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089001:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575913.089004:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575913.089007:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089010:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575913.089031:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575913.089035:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575913.089038:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089040:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575913.089042:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575913.089058:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575913.089063:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575913.089067:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575913.089068:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089072:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575913.089075:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575913.089079:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575913.089087:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575913.089094:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575913.089103:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516881792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575913.089123:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575913.089127:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575913.089132:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516881792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575913.089154:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575913.089158:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575913.089161:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575913.089163:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575913.089164:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575913.089167:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575913.089170:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16600 00000400:00000200:0.0:1788575913.089174:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575913.089177:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575913.089179:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575913.089189:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089192:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575913.089199:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089201:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575913.089204:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575913.089205:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575913.089207:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575913.089209:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575913.089211:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575913.089213:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575913.089215:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089217:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575913.089219:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089220:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575913.089222:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575913.089225:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089227:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575913.089229:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575913.089234:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575913.089236:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089239:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575913.089243:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575913.089246:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089247:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575913.089251:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575913.089253:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575913.089256:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575913.089258:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089259:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575913.089261:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575913.089263:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575913.089274:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575913.089279:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575913.089284:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516881920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575913.089296:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575913.089298:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575913.089302:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516881920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575913.089316:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575913.089319:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575913.089322:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575913.089323:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575913.089325:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575913.089327:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575913.089330:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16600 00000400:00000200:0.0:1788575913.089334:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575913.089337:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575913.089339:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575913.089342:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089343:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575913.089345:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089347:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575913.089349:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575913.089350:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575913.089352:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575913.089354:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575913.089355:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575913.089357:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575913.089360:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089362:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575913.089364:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089366:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575913.089372:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575913.089375:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089378:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575913.089379:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575913.089382:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575913.089383:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089385:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575913.089387:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575913.089390:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089391:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575913.089394:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575913.089396:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575913.089398:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575913.089400:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089401:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575913.089403:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575913.089405:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575913.089408:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575913.089411:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575913.089415:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516882048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575913.089422:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575913.089424:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575913.089428:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516882048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575913.089435:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575913.089438:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575913.089441:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575913.089443:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575913.089444:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575913.089446:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575913.089453:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16600 00000400:00000200:0.0:1788575913.089456:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575913.089458:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575913.089460:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575913.089463:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089465:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575913.089467:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089469:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575913.089471:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575913.089472:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575913.089473:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575913.089475:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575913.089477:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575913.089479:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575913.089482:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089489:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575913.089490:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089492:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575913.089494:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575913.089497:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089499:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575913.089501:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575913.089504:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575913.089506:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089508:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575913.089510:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575913.089512:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089513:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575913.089516:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575913.089518:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575913.089520:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575913.089525:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089527:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575913.089528:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575913.089530:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575913.089533:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575913.089536:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575913.089540:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516882176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575913.089546:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575913.089548:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575913.089551:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516882176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575913.089559:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575913.089561:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575913.089564:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575913.089566:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575913.089567:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575913.089570:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575913.089572:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16600 00000400:00000200:0.0:1788575913.089576:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575913.089578:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575913.089580:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575913.089582:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089584:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575913.089586:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089589:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575913.089590:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575913.089592:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575913.089593:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575913.089594:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575913.089596:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575913.089601:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575913.089605:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089606:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575913.089608:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089610:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575913.089612:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575913.089614:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575913.089616:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575913.089618:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575913.089620:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575913.089622:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089624:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575913.089625:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575913.089628:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089629:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575913.089632:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575913.089634:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575913.089636:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575913.089637:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.089639:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575913.089641:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575913.089642:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575913.089645:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788575913.089648:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575913.089652:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516882304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575913.089659:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575913.089661:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575913.089664:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516882304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575913.089671:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575913.089679:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16610) 00000100:00000001:0.0:1788575913.600955:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575913.600962:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575913.600964:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575913.600970:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575913.600973:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575913.600978:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575913.600983:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575913.600987:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575913.600992:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575913.601066:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575913.601069:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575913.601070:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575913.601072:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575913.601075:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575913.601077:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575913.606866:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575913.606870:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575913.607771:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575914.304979:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575914.304982:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575914.304985:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575914.624942:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575914.624997:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575914.625141:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575914.625149:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575914.625155:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575914.625162:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575914.625180:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575914.625183:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575914.625186:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575914.625458:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575914.625462:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575914.625464:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575914.625467:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575914.625470:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575914.625472:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575914.625474:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575914.625487:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575914.625490:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575915.330052:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575915.330056:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575915.330059:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575915.649048:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575915.649053:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575915.649056:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575915.649060:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575915.649066:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575915.649070:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575915.649076:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575915.649078:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575915.649081:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575915.650873:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575915.650878:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575915.650880:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575915.650883:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575915.650886:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575915.650888:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575915.650890:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575915.650892:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575915.650894:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1788575916.161850:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575916.161868:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575916.161877:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575916.161880:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788575916.353179:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575916.353182:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575916.353185:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575916.672967:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575916.672973:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575916.672977:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575916.672984:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575916.672990:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575916.672995:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575916.673012:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575916.673015:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575916.673037:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575916.928912:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311276781, at 4311276800 00000100:00000010:0.0:1788575916.928927:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf420. 00000020:00000001:0.0:1788575917.377077:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575917.377081:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575917.377084:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575917.696935:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575917.696945:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575917.696948:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575917.696958:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575917.696964:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575917.696969:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575917.696974:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575917.696978:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575917.696987:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.209116:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575918.209122:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575918.209127:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575918.209138:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4a80 x1875440516882432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.209155:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575918.209158:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575918.209163:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575918.209167:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575918.209173:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575918.209180:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516882432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575918.209187:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575918.209192:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575918.209197:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575918.209201:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575918.209208:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575918.209212:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575918.209220:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575918.209226:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575918.209237:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575918.209259:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516882432, portal 25 00000100:00000001:0.0:1788575918.209263:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575918.209276:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845056 : -123400582706560 : ffff8fc4901a4a80) 00000100:00000040:0.0:1788575918.209283:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4a80 x1875440516882432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.209295:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575918.209299:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575918.209305:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575918.209311:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516882432, offset 0 00000400:00000200:0.0:1788575918.209322:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575918.209334:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575918.209349:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575918.209358:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575918.209366:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575918.209373:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575918.209377:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575918.209383:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548546:548546:256:4294967295] 192.168.202.128@tcp LPNI seq info [548469:548469:8:4294967295] 00000400:00000200:0.0:1788575918.209394:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575918.209411:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575918.209420:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575918.209428:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575918.209437:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575918.209443:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575918.209470:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575918.209474:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575918.209481:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575918.209488:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575918.209495:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575918.209499:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575918.209506:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575918.209514:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4a80 x1875440516882432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.209536:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575918.209542:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4a80 x1875440516882432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.209553:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.209556:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575918.209562:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575918.209568:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575918.209572:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311281901 00000100:00000010:0.0:1788575918.209578:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50840. 00000100:00000001:0.0:1788575918.209585:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575918.209591:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311281901 00000100:00000010:0.0:1788575918.209594:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50360. 00000100:00000001:0.0:1788575918.209598:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575918.209601:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575918.209604:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.209609:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575918.209616:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575918/real 1788575918] req@ffff8fc4901a4a80 x1875440516882432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575918.209626:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575918.209628:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575918.209631:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575918.209634:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575918.209637:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575918.209640:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575918.209644:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4a80 x1875440516882432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575918.209654:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4a80 x1875440516882432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.209662:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575918.209668:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.209672:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575918.209675:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575918.209681:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4a80 x1875440516882432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.209701:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575918.209706:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4a80 x1875440516882432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.209717:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575918.209722:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.209725:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575918.209727:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.209731:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575918.209734:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575918.209738:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575918.209744:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575918.209747:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575918.209750:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4a80 x1875440516882432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575918.209771:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516882432:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575918.209777:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.209779:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575918.209782:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4a80 x1875440516882432/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.209789:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575918.209791:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575918.209794:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575918.209797:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575918.209799:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575918.209805:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575918.209821:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575918.209823:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6800. 02000000:00000001:0.0:1788575918.209826:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575918.209828:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575918.209829:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4a80. 00000100:00000001:0.0:1788575918.209833:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575918.209834:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.209835:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.209840:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575918.209841:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.209843:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575918.209845:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575918.209847:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575918.209853:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7100 x1875440516882560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.209863:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575918.209866:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575918.209871:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575918.209875:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575918.209880:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575918.209888:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516882560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575918.209895:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575918.209899:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575918.209903:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575918.209908:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575918.209913:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575918.209917:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575918.209923:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575918.209927:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575918.209931:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575918.209935:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516882560, portal 10 00000100:00000001:0.0:1788575918.209938:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575918.209941:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126854912 : -123400582696704 : ffff8fc4901a7100) 00000100:00000040:0.0:1788575918.209948:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7100 x1875440516882560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.209968:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575918.209971:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575918.209975:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575918.209981:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516882560, offset 0 00000400:00000200:0.0:1788575918.209987:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575918.209992:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575918.209998:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575918.210002:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575918.210008:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575918.210015:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575918.210020:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575918.210027:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548547:548547:256:4294967295] 192.168.202.128@tcp LPNI seq info [548470:548470:8:4294967295] 00000400:00000200:0.0:1788575918.210037:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575918.210052:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575918.210059:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575918.210063:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575918.210068:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575918.210070:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575918.210079:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575918.210081:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575918.210083:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575918.210087:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575918.210090:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575918.210091:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575918.210094:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575918.210098:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7100 x1875440516882560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210104:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575918.210108:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7100 x1875440516882560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210119:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.210120:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575918.210123:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575918.210126:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575918.210129:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575918.210130:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311278099 00000100:00000001:0.0:1788575918.210132:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311281901 00000100:00000001:0.0:1788575918.210133:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311278099 00000100:00000001:0.0:1788575918.210135:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311281901 00000100:00000001:0.0:1788575918.210136:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575918.210137:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.210138:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575918.210140:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575918/real 1788575918] req@ffff8fc4901a7100 x1875440516882560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575918.210145:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575918.210145:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575918.210146:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575918.210148:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575918.210149:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575918.210150:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575918.210152:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7100 x1875440516882560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575918.210156:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7100 x1875440516882560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210160:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575918.210161:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210162:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575918.210163:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575918.210165:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7100 x1875440516882560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210171:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575918.210174:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7100 x1875440516882560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210178:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575918.210179:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210180:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575918.210181:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210181:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575918.210183:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575918.210184:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575918.210186:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575918.210188:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575918.210190:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7100 x1875440516882560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575918.210194:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516882560:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575918.210196:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575918.210198:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5180 x1875440516882688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210201:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575918.210202:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575918.210203:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575918.210204:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575918.210205:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575918.210207:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516882688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575918.210209:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575918.210210:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575918.210213:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575918.210214:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575918.210216:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575918.210218:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575918.210220:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575918.210221:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575918.210223:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575918.210225:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516882688, portal 10 00000100:00000001:0.0:1788575918.210226:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575918.210227:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126846848 : -123400582704768 : ffff8fc4901a5180) 00000100:00000040:0.0:1788575918.210231:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5180 x1875440516882688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210235:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575918.210236:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575918.210237:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575918.210239:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516882688, offset 0 00000400:00000200:0.0:1788575918.210241:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575918.210243:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575918.210245:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575918.210247:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575918.210248:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575918.210250:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575918.210251:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575918.210253:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548548:548548:256:4294967295] 192.168.202.128@tcp LPNI seq info [548471:548471:8:4294967295] 00000400:00000200:0.0:1788575918.210255:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575918.210259:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575918.210262:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575918.210263:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575918.210266:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575918.210267:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575918.210273:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575918.210277:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575918.210279:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575918.210281:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575918.210283:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575918.210284:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575918.210285:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575918.210287:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5180 x1875440516882688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210292:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575918.210294:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5180 x1875440516882688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210297:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.210299:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575918.210301:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575918.210304:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575918.210306:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575918.210308:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311278106 00000100:00000001:0.0:1788575918.210310:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311281901 00000100:00000001:0.0:1788575918.210312:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311278106 00000100:00000001:0.0:1788575918.210314:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311281901 00000100:00000001:0.0:1788575918.210316:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575918.210318:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.210319:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575918.210323:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575918/real 1788575918] req@ffff8fc4901a5180 x1875440516882688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575918.210331:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575918.210332:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575918.210333:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575918.210336:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575918.210337:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575918.210344:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575918.210347:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5180 x1875440516882688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575918.210355:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5180 x1875440516882688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210361:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575918.210363:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210364:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575918.210366:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575918.210369:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5180 x1875440516882688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210375:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575918.210380:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5180 x1875440516882688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210386:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575918.210388:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210390:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575918.210391:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210393:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575918.210395:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575918.210397:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575918.210401:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575918.210403:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575918.210406:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5180 x1875440516882688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575918.210414:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516882688:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575918.210418:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575918.210426:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5c00 x1875440516882816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210432:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575918.210433:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575918.210439:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575918.210440:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575918.210441:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575918.210443:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516882816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575918.210445:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575918.210446:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575918.210447:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575918.210449:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575918.210451:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575918.210452:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575918.210454:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575918.210456:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575918.210458:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575918.210460:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516882816, portal 4 00000100:00000001:0.0:1788575918.210461:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575918.210462:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126849536 : -123400582702080 : ffff8fc4901a5c00) 00000100:00000040:0.0:1788575918.210466:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5c00 x1875440516882816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210470:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575918.210473:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575918.210475:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575918.210477:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516882816, offset 0 00000400:00000200:0.0:1788575918.210481:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575918.210484:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575918.210487:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575918.210490:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575918.210492:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575918.210495:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575918.210497:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575918.210506:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548549:548549:256:4294967295] 192.168.202.128@tcp LPNI seq info [548472:548472:8:4294967295] 00000400:00000200:0.0:1788575918.210510:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575918.210516:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575918.210520:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575918.210524:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575918.210527:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575918.210530:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575918.210539:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575918.210541:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575918.210544:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575918.210547:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575918.210550:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575918.210551:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575918.210554:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575918.210557:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5c00 x1875440516882816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210562:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575918.210564:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5c00 x1875440516882816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210568:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.210569:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575918.210571:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575918.210573:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575918.210575:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575918.210576:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311278246 00000100:00000001:0.0:1788575918.210577:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311281901 00000100:00000001:0.0:1788575918.210579:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311278246 00000100:00000001:0.0:1788575918.210580:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311281901 00000100:00000001:0.0:1788575918.210581:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575918.210586:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.210587:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575918.210589:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575918/real 1788575918] req@ffff8fc4901a5c00 x1875440516882816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575918.210595:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575918.210596:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575918.210597:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575918.210598:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575918.210599:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575918.210601:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575918.210602:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5c00 x1875440516882816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575918.210607:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5c00 x1875440516882816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210610:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575918.210611:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210612:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575918.210613:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575918.210615:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5c00 x1875440516882816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210618:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575918.210620:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5c00 x1875440516882816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210624:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575918.210625:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210626:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575918.210626:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210627:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575918.210628:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575918.210630:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575918.210643:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575918.210645:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575918.210647:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5c00 x1875440516882816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575918.210651:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516882816:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575918.210653:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210654:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575918.210656:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7100 x1875440516882560/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210660:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575918.210660:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575918.210662:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575918.210663:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575918.210664:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575918.210665:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575918.210666:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575918.210667:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4800. 02000000:00000001:0.0:1788575918.210669:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575918.210670:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575918.210671:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7100. 00000100:00000001:0.0:1788575918.210673:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575918.210674:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210675:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575918.210676:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5180 x1875440516882688/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210680:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575918.210681:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575918.210681:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9cc00. 02000000:00000001:0.0:1788575918.210682:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575918.210683:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575918.210684:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575918.210685:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575918.210702:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7c00. 02000000:00000001:0.0:1788575918.210704:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575918.210705:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575918.210706:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5180. 00000100:00000001:0.0:1788575918.210709:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575918.210709:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210711:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575918.210713:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5c00 x1875440516882816/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210717:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575918.210718:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575918.210718:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575918.210720:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575918.210720:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575918.210721:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575918.210722:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575918.210723:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575918.210724:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575918.210724:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575918.210725:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5c00. 00000100:00000001:0.0:1788575918.210726:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575918.210726:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210727:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210730:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575918.210730:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.210731:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575918.210732:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575918.210733:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575918.210735:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7800 x1875440516882944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210738:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575918.210739:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575918.210740:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575918.210741:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575918.210743:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575918.210744:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516882944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575918.210749:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575918.210750:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575918.210751:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575918.210752:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575918.210754:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575918.210756:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575918.210759:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575918.210761:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575918.210764:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575918.210767:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516882944, portal 4 00000100:00000001:0.0:1788575918.210769:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575918.210770:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126856704 : -123400582694912 : ffff8fc4901a7800) 00000100:00000040:0.0:1788575918.210775:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7800 x1875440516882944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210782:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575918.210784:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575918.210786:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575918.210788:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516882944, offset 0 00000400:00000200:0.0:1788575918.210792:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575918.210796:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575918.210799:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575918.210801:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575918.210804:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575918.210806:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575918.210826:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575918.210829:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548550:548550:256:4294967295] 192.168.202.128@tcp LPNI seq info [548473:548473:8:4294967295] 00000400:00000200:0.0:1788575918.210833:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575918.210838:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575918.210843:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575918.210846:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575918.210849:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575918.210857:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1788575918.210867:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575918.210869:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575918.210872:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575918.210875:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575918.210878:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575918.210880:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575918.210882:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575918.210885:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7800 x1875440516882944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210890:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575918.210891:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7800 x1875440516882944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575943 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210895:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.210896:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575918.210898:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575918.210900:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575918.210902:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575918.210903:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311278249 00000100:00000001:0.0:1788575918.210905:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311281901 00000100:00000001:0.0:1788575918.210906:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311278249 00000100:00000001:0.0:1788575918.210907:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311281901 00000100:00000001:0.0:1788575918.210908:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575918.210910:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.210911:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575918.210913:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575918/real 1788575918] req@ffff8fc4901a7800 x1875440516882944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575918.210917:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575918.210918:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575918.210923:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575918.210924:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575918.210925:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575918.210927:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575918.210929:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7800 x1875440516882944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575918.210934:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7800 x1875440516882944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210937:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575918.210938:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210939:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575918.210940:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575918.210941:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7800 x1875440516882944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210945:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575918.210947:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7800 x1875440516882944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210951:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575918.210952:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210953:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575918.210954:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210955:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575918.210956:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575918.210957:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575918.210959:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575918.210960:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575918.210962:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7800 x1875440516882944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575918.210966:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516882944:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575918.210971:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210972:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575918.210974:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7800 x1875440516882944/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575943 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575918.210977:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575918.210978:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575918.210979:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c000. 02000000:00000001:0.0:1788575918.210980:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575918.210981:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575918.210982:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575918.210983:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575918.210984:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575918.210985:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575918.210986:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575918.210987:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7800. 00000100:00000001:0.0:1788575918.210988:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575918.210989:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210990:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.210992:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575918.210993:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.210995:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575918.210996:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575918.720927:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575918.720938:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575918.720943:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575918.720951:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575918.720958:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575918.720964:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.721038:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575918.721040:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575918.721042:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575919.425013:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575919.425017:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575919.425020:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575919.744922:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575919.744929:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575919.744950:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575919.744954:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575919.744959:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575919.744964:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575919.744995:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575919.744998:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575919.745002:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575920.449026:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575920.449029:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575920.449031:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575920.768926:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575920.768934:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575920.768937:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575920.768942:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575920.768946:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575920.768949:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575920.768963:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575920.768965:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575920.768969:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575920.769064:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575920.769068:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575920.769071:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575920.769073:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575920.769077:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575920.769082:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575920.769086:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575920.769088:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575920.769091:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575921.473048:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575921.473051:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575921.473054:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575922.048915:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311281901, at 4311281920 00000100:00000010:0.0:1788575922.048931:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50840. 00000020:00000001:0.0:1788575922.497032:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575922.497035:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575922.497037:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575922.817037:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575922.817056:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575922.817057:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575922.817060:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575922.817062:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575922.817064:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575922.817066:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575922.817067:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575922.817069:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575922.819125:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575922.819129:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575922.819134:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575922.819140:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575922.819143:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575922.819149:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575922.819155:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575922.819159:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575922.819163:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575923.329131:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575923.329141:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575923.329144:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575923.329148:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575923.329152:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575923.329161:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16610 00000400:00000200:0.0:1788575923.329169:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575923.329174:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575923.329177:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575923.329180:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329183:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575923.329185:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329188:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575923.329192:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575923.329194:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575923.329196:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575923.329199:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575923.329216:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575923.329218:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575923.329229:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329231:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575923.329233:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329234:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575923.329237:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575923.329241:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329243:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575923.329246:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575923.329249:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575923.329251:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.329254:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575923.329256:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575923.329261:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575923.329264:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575923.329268:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575923.329270:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.329273:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575923.329275:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575923.329279:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575923.329286:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575923.329292:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575923.329300:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516883072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575923.329310:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575923.329314:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575923.329323:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516883072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575923.329347:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575923.329350:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575923.329365:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575923.329366:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575923.329368:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575923.329371:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575923.329375:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16610 00000400:00000200:0.0:1788575923.329379:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575923.329382:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575923.329387:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575923.329391:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329395:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575923.329400:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329403:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575923.329406:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575923.329408:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575923.329411:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575923.329414:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575923.329416:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575923.329418:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575923.329424:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329427:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575923.329431:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329436:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575923.329442:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575923.329446:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329449:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575923.329454:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575923.329461:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575923.329465:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.329469:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575923.329476:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575923.329485:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.329489:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575923.329494:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575923.329499:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575923.329503:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575923.329505:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.329508:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575923.329511:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575923.329513:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575923.329520:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575923.329524:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575923.329533:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516883200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575923.329546:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575923.329550:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575923.329560:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516883200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575923.329577:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575923.329582:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575923.329588:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575923.329591:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575923.329595:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575923.329600:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575923.329606:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16610 00000400:00000200:0.0:1788575923.329615:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575923.329622:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575923.329627:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575923.329632:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329637:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575923.329655:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329668:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575923.329672:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575923.329674:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575923.329676:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575923.329678:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575923.329683:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575923.329697:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575923.329704:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329708:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575923.329713:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329718:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575923.329721:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575923.329726:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329731:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575923.329734:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575923.329737:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575923.329743:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.329754:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575923.329758:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575923.329762:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.329767:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575923.329773:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575923.329778:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575923.329782:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575923.329784:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.329789:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575923.329792:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575923.329796:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575923.329802:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575923.329840:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575923.329848:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516883328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575923.329872:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575923.329877:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575923.329883:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516883328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575923.329896:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575923.329909:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575923.329915:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575923.329917:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575923.329921:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575923.329925:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575923.329930:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16610 00000400:00000200:0.0:1788575923.329937:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575923.329943:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575923.329948:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575923.329960:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329963:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575923.329965:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329966:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575923.329969:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575923.329970:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575923.329971:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575923.329974:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575923.329975:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575923.329977:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575923.329980:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329982:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575923.329984:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329986:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575923.329992:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575923.329995:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575923.329998:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575923.329999:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575923.330002:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575923.330005:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.330007:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575923.330009:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575923.330011:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.330012:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575923.330015:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575923.330017:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575923.330020:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575923.330021:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.330023:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575923.330024:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575923.330026:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575923.330029:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788575923.330032:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575923.330037:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516883456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575923.330044:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575923.330046:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575923.330050:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516883456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575923.330061:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575923.330065:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575923.330070:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575923.330074:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575923.330076:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575923.330081:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575923.330090:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16610 00000400:00000200:0.0:1788575923.330097:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575923.330104:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575923.330108:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575923.330113:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.330118:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575923.330122:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.330126:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575923.330132:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575923.330134:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575923.330137:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575923.330140:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575923.330145:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575923.330150:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575923.330156:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.330162:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575923.330166:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575923.330169:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575923.330175:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575923.330179:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575923.330183:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575923.330187:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575923.330191:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575923.330196:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.330200:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575923.330204:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575923.330208:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.330212:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575923.330218:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575923.330222:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575923.330225:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575923.330231:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575923.330233:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575923.330237:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575923.330241:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575923.330248:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788575923.330254:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575923.330262:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516883584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575923.330275:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575923.330278:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575923.330285:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516883584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575923.330310:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575923.330316:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16620) 00000100:00000001:0.0:1788575923.840953:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575923.840963:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575923.840968:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575923.840978:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575923.840984:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575923.840990:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575923.841012:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575923.841013:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575923.841016:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575924.545113:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575924.545117:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575924.545120:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575924.865116:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575924.865119:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575924.865121:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575924.865124:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575924.865126:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575924.865128:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575924.865203:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575924.865206:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575924.865209:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575925.569026:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575925.569030:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575925.569033:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575925.888982:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575925.888986:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575925.888988:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575925.888991:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575925.888993:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575925.888995:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575925.889644:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575925.889648:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575925.889651:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575926.593006:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575926.593010:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575926.593014:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575926.912941:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575926.912946:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575926.912948:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575926.912953:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575926.912958:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575926.912961:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575926.912964:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575926.912967:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575926.912970:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575926.912997:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575926.913002:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575926.913006:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575926.913011:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575926.913016:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575926.913021:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575926.913044:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575926.913046:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575926.913048:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575927.168984:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311287021, at 4311287040 00000100:00000010:0.0:1788575927.168988:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf360. 00000020:00000001:0.0:1788575927.617036:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575927.617044:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575927.617071:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575927.937117:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575927.937121:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575927.937125:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575927.937129:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575927.937135:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575927.937140:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575927.937257:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575927.937259:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575927.937262:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.449409:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575928.449421:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575928.449436:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575928.449456:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a5880 x1875440516883712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.449475:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575928.449480:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575928.449488:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575928.449494:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575928.449501:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575928.449510:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516883712:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575928.449527:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575928.449536:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575928.449541:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575928.449547:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575928.449559:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575928.449565:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575928.449579:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575928.449586:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575928.449612:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575928.449626:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516883712, portal 25 00000100:00000001:0.0:1788575928.449635:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575928.449639:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126848640 : -123400582702976 : ffff8fc4901a5880) 00000100:00000040:0.0:1788575928.449653:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a5880 x1875440516883712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.449737:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575928.449748:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575928.449752:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575928.449760:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875440516883712, offset 0 00000400:00000200:0.0:1788575928.449776:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575928.449794:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575928.449832:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575928.449841:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575928.449851:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575928.449860:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575928.449867:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575928.449879:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548556:548556:256:4294967295] 192.168.202.128@tcp LPNI seq info [548479:548479:8:4294967295] 00000400:00000200:0.0:1788575928.449892:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575928.449918:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575928.449934:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575928.449946:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575928.449959:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575928.449971:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575928.450021:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575928.450030:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575928.450041:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575928.450058:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575928.450069:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575928.450077:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575928.450099:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575928.450111:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a5880 x1875440516883712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.450144:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575928.450155:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a5880 x1875440516883712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.450211:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.450219:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575928.450234:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575928.450244:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575928.450250:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 0 is 4311292142 00000100:00000010:0.0:1788575928.450259:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50840. 00000100:00000001:0.0:1788575928.450271:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788575928.450284:0:2398:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS new deadline estimation for cpu 1 is 4311292142 00000100:00000010:0.0:1788575928.450290:0:2398:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8fc486b50d80. 00000100:00000001:0.0:1788575928.450301:0:2398:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.128@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788575928.450307:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575928.450313:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.450325:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575928.450340:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575928/real 1788575928] req@ffff8fc4901a5880 x1875440516883712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575928.450384:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575928.450390:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575928.450397:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575928.450403:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575928.450411:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575928.450418:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575928.450429:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a5880 x1875440516883712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575928.450469:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a5880 x1875440516883712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.450499:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575928.450504:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.450513:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575928.450518:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575928.450531:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a5880 x1875440516883712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.450564:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575928.450578:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a5880 x1875440516883712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.450618:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575928.450628:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.450636:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575928.450639:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.450651:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575928.450657:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575928.450665:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575928.450677:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575928.450685:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575928.450700:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a5880 x1875440516883712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575928.450723:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3d5a86eb-f614-41e3-98bd-af00951045c6:2398:1875440516883712:192.168.202.128@tcp:250:kworker.0 00000100:00000001:0.0:1788575928.450732:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.450735:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575928.450741:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a5880 x1875440516883712/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.450751:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575928.450755:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575928.450759:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575928.450764:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575928.450767:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575928.450769:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4acae6800 refcount=3 obd=MGC192.168.202.128@tcp 00000020:00000001:0.0:1788575928.450772:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575928.450776:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e6c00. 02000000:00000001:0.0:1788575928.450781:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575928.450789:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575928.450792:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a5880. 00000100:00000001:0.0:1788575928.450799:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575928.450800:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.450804:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.450838:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575928.450842:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.450845:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575928.450847:0:2398:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575928.450849:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575928.450856:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4700 x1875440516883840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.450864:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575928.450866:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575928.450869:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575928.450872:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575928.450874:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575928.450877:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516883840:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575928.450881:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575928.450883:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575928.450885:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575928.450887:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575928.450890:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575928.450892:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575928.450896:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575928.450900:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575928.450904:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575928.450907:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516883840, portal 10 00000100:00000001:0.0:1788575928.450909:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575928.450911:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126844160 : -123400582707456 : ffff8fc4901a4700) 00000100:00000040:0.0:1788575928.450917:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4700 x1875440516883840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.450924:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575928.450927:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575928.450933:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575928.450936:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516883840, offset 0 00000400:00000200:0.0:1788575928.450940:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575928.450944:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575928.450949:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575928.450952:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575928.450955:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575928.450958:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575928.450960:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575928.450964:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548557:548557:256:4294967295] 192.168.202.128@tcp LPNI seq info [548480:548480:8:4294967295] 00000400:00000200:0.0:1788575928.450968:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575928.450975:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575928.450980:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575928.450983:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575928.450988:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575928.450991:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575928.451003:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575928.451005:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575928.451008:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575928.451011:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575928.451015:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575928.451016:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575928.451019:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575928.451023:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4700 x1875440516883840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451031:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575928.451034:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4700 x1875440516883840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451041:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.451042:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575928.451051:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575928.451055:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: using a timeout of 17801 usecs (18 jiffies) 00000400:00000200:0.0:1788575928.451058:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575928.451060:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311288340 00000100:00000001:0.0:1788575928.451063:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311292142 00000100:00000001:0.0:1788575928.451065:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311288340 00000100:00000001:0.0:1788575928.451068:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311292142 00000100:00000001:0.0:1788575928.451070:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575928.451072:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.451074:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575928.451077:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575928/real 1788575928] req@ffff8fc4901a4700 x1875440516883840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575928.451086:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575928.451087:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575928.451089:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575928.451091:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575928.451093:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575928.451095:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575928.451098:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4700 x1875440516883840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575928.451107:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4700 x1875440516883840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451113:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575928.451114:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.451116:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575928.451118:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575928.451121:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4700 x1875440516883840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451128:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575928.451131:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4700 x1875440516883840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451144:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575928.451147:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.451148:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575928.451149:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.451152:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575928.451154:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575928.451156:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575928.451160:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575928.451162:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575928.451166:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4700 x1875440516883840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575928.451174:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516883840:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575928.451178:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575928.451181:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a7480 x1875440516883968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451188:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575928.451189:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575928.451192:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575928.451193:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575928.451195:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575928.451198:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516883968:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575928.451202:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575928.451204:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575928.451205:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575928.451207:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575928.451211:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575928.451212:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575928.451221:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575928.451224:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575928.451227:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575928.451230:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516883968, portal 10 00000100:00000001:0.0:1788575928.451232:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575928.451233:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126855808 : -123400582695808 : ffff8fc4901a7480) 00000100:00000040:0.0:1788575928.451239:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a7480 x1875440516883968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451246:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575928.451248:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575928.451250:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575928.451252:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875440516883968, offset 0 00000400:00000200:0.0:1788575928.451256:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575928.451260:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575928.451263:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575928.451266:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575928.451269:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575928.451271:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575928.451274:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575928.451277:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548558:548558:256:4294967295] 192.168.202.128@tcp LPNI seq info [548481:548481:8:4294967295] 00000400:00000200:0.0:1788575928.451281:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575928.451287:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575928.451292:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575928.451295:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575928.451298:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575928.451301:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575928.451312:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575928.451314:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575928.451316:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575928.451320:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575928.451323:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575928.451329:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575928.451332:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575928.451336:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a7480 x1875440516883968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451344:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575928.451347:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a7480 x1875440516883968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451354:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.451355:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575928.451358:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575928.451362:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: using a timeout of 24691 usecs (25 jiffies) 00000400:00000200:0.0:1788575928.451365:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575928.451367:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311288347 00000100:00000001:0.0:1788575928.451369:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311292142 00000100:00000001:0.0:1788575928.451371:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311288347 00000100:00000001:0.0:1788575928.451373:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311292142 00000100:00000001:0.0:1788575928.451376:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575928.451378:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.451380:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575928.451383:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575928/real 1788575928] req@ffff8fc4901a7480 x1875440516883968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575928.451392:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575928.451393:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575928.451395:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575928.451397:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575928.451399:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575928.451402:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575928.451404:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a7480 x1875440516883968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575928.451412:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a7480 x1875440516883968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451425:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575928.451426:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.451428:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575928.451430:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575928.451434:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a7480 x1875440516883968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451440:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575928.451444:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a7480 x1875440516883968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451450:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575928.451453:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.451454:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575928.451455:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.451457:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575928.451460:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575928.451463:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575928.451466:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575928.451469:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575928.451473:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a7480 x1875440516883968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575928.451481:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516883968:192.168.202.128@tcp:38:kworker.0 00000100:00000001:0.0:1788575928.451485:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575928.451489:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a6300 x1875440516884096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451496:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575928.451498:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575928.451504:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575928.451506:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575928.451508:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575928.451526:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516884096:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575928.451530:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575928.451532:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575928.451534:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575928.451535:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575928.451539:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575928.451540:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575928.451544:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575928.451547:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575928.451550:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575928.451553:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516884096, portal 4 00000100:00000001:0.0:1788575928.451555:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575928.451557:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126851328 : -123400582700288 : ffff8fc4901a6300) 00000100:00000040:0.0:1788575928.451563:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a6300 x1875440516884096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451569:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575928.451571:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575928.451574:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575928.451576:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516884096, offset 0 00000400:00000200:0.0:1788575928.451580:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575928.451584:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575928.451587:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575928.451601:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575928.451604:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575928.451606:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575928.451608:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575928.451611:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548559:548559:256:4294967295] 192.168.202.128@tcp LPNI seq info [548482:548482:8:4294967295] 00000400:00000200:0.0:1788575928.451615:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575928.451621:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575928.451630:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575928.451633:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575928.451637:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575928.451640:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575928.451650:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575928.451652:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575928.451654:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575928.451658:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575928.451661:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575928.451663:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575928.451666:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575928.451670:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a6300 x1875440516884096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451677:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575928.451680:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a6300 x1875440516884096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451686:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.451688:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575928.451691:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575928.451694:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: using a timeout of 164072 usecs (165 jiffies) 00000400:00000200:0.0:1788575928.451698:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575928.451699:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311288487 00000100:00000001:0.0:1788575928.451702:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311292142 00000100:00000001:0.0:1788575928.451704:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311288487 00000100:00000001:0.0:1788575928.451706:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311292142 00000100:00000001:0.0:1788575928.451708:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575928.451710:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.451711:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575928.451715:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575928/real 1788575928] req@ffff8fc4901a6300 x1875440516884096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575928.451727:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575928.451728:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575928.451730:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575928.451732:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575928.451734:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575928.451736:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575928.451739:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a6300 x1875440516884096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575928.451747:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a6300 x1875440516884096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451753:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575928.451754:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.451756:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575928.451757:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575928.451760:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a6300 x1875440516884096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451767:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575928.451770:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a6300 x1875440516884096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451777:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575928.451779:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.451781:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575928.451782:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.451784:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575928.451786:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575928.451789:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575928.451792:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575928.451794:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575928.451802:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a6300 x1875440516884096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575928.451833:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516884096:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575928.451837:0:2398:0:(client.c:1821:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788575928.451841:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8fc4901a4e00 x1875440516884224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451847:0:2398:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575928.451849:0:2398:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575928.451852:0:2398:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788575928.451853:0:2398:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575928.451855:0:2398:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575928.451858:0:2398:0:(client.c:1927:ptlrpc_send_new_req()) Sending RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516884224:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575928.451861:0:2398:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788575928.451863:0:2398:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788575928.451864:0:2398:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575928.451866:0:2398:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788575928.451869:0:2398:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575928.451871:0:2398:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788575928.451874:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000010:0.0:1788575928.451877:0:2398:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8fc4b19155d8. 00000400:00000010:0.0:1788575928.451880:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9440. 00000100:00000200:0.0:1788575928.451883:0:2398:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875440516884224, portal 4 00000100:00000001:0.0:1788575928.451885:0:2398:0:(client.c:3283:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788575928.451886:0:2398:0:(client.c:3285:ptlrpc_request_addref()) Process leaving (rc=18446620673126845952 : -123400582705664 : ffff8fc4901a4e00) 00000100:00000040:0.0:1788575928.451891:0:2398:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8fc4901a4e00 x1875440516884224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.451898:0:2398:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788575928.451900:0:2398:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.128@tcp 00000400:00000010:0.0:1788575928.451902:0:2398:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8fc490ed9660. 00000100:00000200:0.0:1788575928.451904:0:2398:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875440516884224, offset 0 00000400:00000200:0.0:1788575928.451914:0:2398:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.202.128@tcp 00000400:00000200:0.0:1788575928.451918:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575928.451921:0:2398:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.128@tcp local destination 00000400:00000200:0.0:1788575928.451924:0:2398:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788575928.451927:0:2398:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.202.28@tcp 00000400:00000200:0.0:1788575928.451929:0:2398:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.28@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788575928.451931:0:2398:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.128@tcp 00000400:00000200:0.0:1788575928.451934:0:2398:0:(lib-move.c:1738:lnet_handle_send()) 192.168.202.28@tcp NI seq info: [548560:548560:256:4294967295] 192.168.202.128@tcp LPNI seq info [548483:548483:8:4294967295] 00000400:00000200:0.0:1788575928.451938:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575928.451945:0:2398:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.202.28@tcp(192.168.202.28@tcp:) -> 192.168.202.128@tcp(192.168.202.128@tcp:192.168.202.128@tcp) : PUT try# 0 00000800:00000200:0.0:1788575928.451950:0:2398:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575928.451953:0:2398:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8fc4b3b56600. 00000800:00000200:0.0:1788575928.451957:0:2398:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc488b0ae40] -> 12345-192.168.202.128@tcp (2) 00000800:00000200:0.0:1788575928.451959:0:2398:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.128 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788575928.451973:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000800:00000010:0.0:1788575928.451975:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8fc4b3b56600. 00000400:00000200:0.0:1788575928.451978:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788575928.451981:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.28@tcp->192.168.202.128@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788575928.451985:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9660 00000400:00000010:0.0:1788575928.451987:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9660. 00000100:00000001:0.0:1788575928.451989:0:2398:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1788575928.451993:0:2398:0:(events.c:40:request_out_callback()) @@@ type 5, status -5 req@ffff8fc4901a4e00 x1875440516884224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.452000:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575928.452003:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8fc4901a4e00 x1875440516884224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 1788575953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.452010:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.452011:0:2398:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788575928.452014:0:2398:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575928.452017:0:2398:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: using a timeout of 166706 usecs (167 jiffies) 00000400:00000200:0.0:1788575928.452020:0:2398:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000100:00000001:0.0:1788575928.452026:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 0 is 4311288490 00000100:00000001:0.0:1788575928.452029:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 0, keep current deadline 4311292142 00000100:00000001:0.0:1788575928.452031:0:2398:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS new deadline estimation for cpu 1 is 4311288490 00000100:00000001:0.0:1788575928.452033:0:2398:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8fc490e72800: PM QoS request active for cpu 1, keep current deadline 4311292142 00000100:00000001:0.0:1788575928.452035:0:2398:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788575928.452037:0:2398:0:(client.c:1949:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.452039:0:2398:0:(client.c:2493:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788575928.452043:0:2398:0:(client.c:2511:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788575928/real 1788575928] req@ffff8fc4901a4e00 x1875440516884224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788575928.452051:0:2398:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788575928.452052:0:2398:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788575928.452054:0:2398:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8fc4b19155d8. 00000400:00000200:0.0:1788575928.452056:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8fc490ed9440 00000400:00000010:0.0:1788575928.452058:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8fc490ed9440. 00000100:00000001:0.0:1788575928.452060:0:2398:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000200:0.0:1788575928.452064:0:2398:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8fc4901a4e00 x1875440516884224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788575928.452072:0:2398:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8fc4901a4e00 x1875440516884224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.452078:0:2398:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788575928.452080:0:2398:0:(client.c:3009:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.452082:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575928.452083:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788575928.452086:0:2398:0:(client.c:2542:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8fc4901a4e00 x1875440516884224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.452093:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788575928.452096:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8fc4901a4e00 x1875440516884224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.452103:0:2398:0:(client.c:2132:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788575928.452110:0:2398:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.452112:0:2398:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788575928.452113:0:2398:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.452115:0:2398:0:(import.c:1101:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788575928.452117:0:2398:0:(import.c:1136:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788575928.452120:0:2398:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788575928.452124:0:2398:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00000001:0.0:1788575928.452126:0:2398:0:(import.c:1561:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788575928.452130:0:2398:0:(lustre_net.h:2424:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8fc4901a4e00 x1875440516884224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788575928.452138:0:2398:0:(client.c:2422:ptlrpc_check_set()) Completed RPC req@ffff8fc4901a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:51d6aedc-ea7c-426a-b495-22e82c13ee75:2398:1875440516884224:192.168.202.128@tcp:8:kworker.0 00000100:00000001:0.0:1788575928.452142:0:2398:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.452144:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575928.452148:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4700 x1875440516883840/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.452155:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575928.452156:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575928.452159:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c400. 02000000:00000001:0.0:1788575928.452161:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575928.452162:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575928.452164:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc485ade000 refcount=5 obd=lustre-MDT0000-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575928.452166:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575928.452168:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4c00. 02000000:00000001:0.0:1788575928.452171:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575928.452173:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575928.452175:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4700. 00000100:00000001:0.0:1788575928.452179:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575928.452181:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.452182:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575928.452186:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a7480 x1875440516883968/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.452203:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575928.452204:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575928.452206:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9c800. 02000000:00000001:0.0:1788575928.452208:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575928.452209:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575928.452211:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc486471000 refcount=9 obd=lustre-MDT0001-mdc-ffff8fc490e72800 00000020:00000001:0.0:1788575928.452213:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575928.452214:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e7800. 02000000:00000001:0.0:1788575928.452217:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575928.452218:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575928.452220:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a7480. 00000100:00000001:0.0:1788575928.452224:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575928.452225:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.452227:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575928.452231:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a6300 x1875440516884096/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.452238:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575928.452239:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575928.452241:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9e000. 02000000:00000001:0.0:1788575928.452244:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575928.452245:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575928.452247:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1000 refcount=5 obd=lustre-OST0000-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575928.452249:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575928.452250:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e4000. 02000000:00000001:0.0:1788575928.452252:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575928.452253:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575928.452255:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a6300. 00000100:00000001:0.0:1788575928.452259:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575928.452260:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.452262:0:2398:0:(client.c:2875:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788575928.452266:0:2398:0:(client.c:2883:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8fc4901a4e00 x1875440516884224/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1788575953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575928.452271:0:2398:0:(client.c:2801:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788575928.452273:0:2398:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788575928.452279:0:2398:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8fc4a6c9dc00. 02000000:00000001:0.0:1788575928.452281:0:2398:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788575928.452283:0:2398:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1788575928.452284:0:2398:0:(genops.c:1223:class_import_put()) import ffff8fc4912d1800 refcount=6 obd=lustre-OST0001-osc-ffff8fc490e72800 00000020:00000001:0.0:1788575928.452286:0:2398:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1788575928.452287:0:2398:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8fc4a66e5400. 02000000:00000001:0.0:1788575928.452290:0:2398:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788575928.452291:0:2398:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788575928.452292:0:2398:0:(client.c:619:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8fc4901a4e00. 00000100:00000001:0.0:1788575928.452296:0:2398:0:(client.c:2857:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788575928.452298:0:2398:0:(client.c:2916:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.452300:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.452305:0:2398:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575928.452306:0:2398:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.452308:0:2398:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788575928.452310:0:2398:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575928.960964:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575928.960971:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575928.960973:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575928.960984:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575928.960988:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575928.960992:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.960995:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575928.960998:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575928.961000:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575929.665043:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575929.665048:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575929.665052:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575929.985003:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575929.985009:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575929.985011:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575929.985016:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575929.985019:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575929.985021:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575929.985023:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575929.985025:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575929.985027:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575930.689012:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575930.689027:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575930.689030:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575931.008991:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575931.008994:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575931.008995:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575931.008997:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575931.008999:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575931.009001:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575931.010047:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575931.010050:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575931.010053:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575931.712981:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575931.712985:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575931.712989:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575932.032928:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575932.032934:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575932.032938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575932.032945:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575932.032951:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575932.032954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575932.033947:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575932.033950:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575932.033960:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575932.288929:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311292142, at 4311292160 00000100:00000010:0.0:1788575932.288952:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc486b50840. 00000020:00000001:0.0:1788575932.736995:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575932.736999:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575932.737002:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575933.056922:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575933.056931:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575933.056936:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575933.056944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575933.056950:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575933.056954:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575933.057589:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575933.057592:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575933.057595:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575933.568936:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575933.568948:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575933.568951:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575933.568955:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575933.568960:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575933.568966:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16621 00000400:00000200:0.0:1788575933.568975:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575933.568981:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575933.568984:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575933.568987:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.568990:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575933.568993:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.568995:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575933.568999:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575933.569001:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575933.569003:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575933.569005:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575933.569008:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575933.569010:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575933.569012:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569014:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575933.569016:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569018:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575933.569021:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575933.569024:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569027:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575933.569029:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575933.569032:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575933.569035:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.569038:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575933.569040:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575933.569062:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575933.569066:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575933.569070:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575933.569071:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.569074:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575933.569076:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575933.569080:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575933.569087:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575933.569093:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575933.569102:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516884352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575933.569111:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575933.569114:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575933.569119:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516884352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575933.569140:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575933.569143:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575933.569147:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575933.569148:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575933.569149:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575933.569152:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575933.569154:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16621 00000400:00000200:0.0:1788575933.569168:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575933.569174:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575933.569178:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575933.569183:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569187:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575933.569191:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569195:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575933.569200:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575933.569202:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575933.569210:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575933.569215:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575933.569219:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575933.569230:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575933.569235:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569236:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575933.569240:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569245:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575933.569248:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575933.569253:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569259:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575933.569263:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575933.569269:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575933.569272:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.569276:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575933.569281:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575933.569285:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.569289:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575933.569296:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575933.569302:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575933.569304:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575933.569305:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.569307:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575933.569310:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575933.569313:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575933.569318:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575933.569330:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575933.569340:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516884480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575933.569353:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575933.569356:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575933.569362:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516884480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575933.569382:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575933.569388:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575933.569394:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575933.569398:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575933.569402:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575933.569407:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575933.569413:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16621 00000400:00000200:0.0:1788575933.569422:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575933.569426:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575933.569428:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575933.569433:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569438:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575933.569444:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569448:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575933.569452:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575933.569454:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575933.569456:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575933.569460:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575933.569464:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575933.569468:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575933.569476:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569480:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575933.569482:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569486:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575933.569489:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575933.569495:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569498:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575933.569508:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575933.569513:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575933.569517:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.569520:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575933.569523:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575933.569528:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.569530:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575933.569536:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575933.569540:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575933.569563:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575933.569565:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.569567:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575933.569568:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575933.569569:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575933.569573:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575933.569579:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575933.569594:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516884608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575933.569607:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575933.569611:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575933.569618:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516884608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575933.569631:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575933.569636:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575933.569640:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575933.569644:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575933.569646:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575933.569649:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575933.569654:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16621 00000400:00000200:0.0:1788575933.569662:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575933.569668:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575933.569671:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575933.569682:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569693:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575933.569697:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569701:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575933.569706:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575933.569709:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575933.569712:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575933.569716:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575933.569721:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575933.569725:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575933.569731:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569735:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575933.569739:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569745:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575933.569748:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575933.569750:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569754:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575933.569756:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575933.569759:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575933.569764:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.569767:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575933.569771:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575933.569776:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.569780:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575933.569786:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575933.569790:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575933.569795:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575933.569798:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.569802:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575933.569804:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575933.569843:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575933.569850:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575933.569861:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575933.569871:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516884736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575933.569885:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575933.569889:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575933.569896:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516884736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575933.569916:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575933.569920:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575933.569925:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575933.569928:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575933.569930:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575933.569936:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575933.569941:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16621 00000400:00000200:0.0:1788575933.569948:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575933.569954:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575933.569959:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575933.569962:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569966:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575933.569971:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569976:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575933.569979:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575933.569982:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575933.569986:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575933.569989:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575933.569992:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575933.569993:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575933.569996:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.569998:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575933.570008:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575933.570013:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575933.570017:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575933.570023:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575933.570028:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575933.570032:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575933.570036:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575933.570039:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.570041:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575933.570042:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575933.570048:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.570052:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575933.570058:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575933.570063:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575933.570068:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575933.570072:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575933.570076:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575933.570079:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575933.570083:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575933.570088:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575933.570094:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575933.570102:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516884864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575933.570117:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575933.570120:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575933.570127:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516884864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575933.570142:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575933.570149:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16631) 00000100:00000001:0.0:1788575934.080919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575934.080927:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575934.080931:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575934.080955:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575934.080959:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575934.080962:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575934.080964:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575934.080965:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575934.080967:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575934.785088:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575934.785092:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575934.785094:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575935.105236:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575935.105240:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575935.105241:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575935.105244:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575935.105246:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575935.105248:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575935.105250:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575935.105251:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575935.105253:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575935.105499:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575935.105502:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575935.105505:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575935.105510:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575935.105516:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575935.105520:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575935.105525:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575935.105528:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575935.105563:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575935.809078:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575935.809082:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575935.809085:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575936.128911:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575936.128918:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575936.128921:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575936.128928:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575936.128932:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575936.128936:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575936.128947:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575936.128950:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575936.128967:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575936.833016:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575936.833020:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575936.833023:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575937.152923:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575937.152933:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575937.152938:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575937.152948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575937.152956:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575937.152962:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575937.152989:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575937.152993:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575937.152997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575937.408928:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311297261, at 4311297280 00000100:00000010:0.0:1788575937.408942:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfcc0. 00000020:00000001:0.0:1788575937.857087:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575937.857091:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575937.857093:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575938.176936:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575938.176940:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575938.176942:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575938.176944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575938.176947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575938.176950:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575938.176953:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575938.176955:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575938.176959:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575939.200942:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575939.200952:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575939.200955:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575939.200963:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575939.200969:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575939.200975:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575939.200980:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575939.200983:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575939.200986:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575939.905071:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575939.905095:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575939.905099:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575940.225017:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575940.225024:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575940.225027:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575940.225031:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575940.225035:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575940.225038:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575940.225041:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575940.225043:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575940.225046:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575940.929127:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575940.929131:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575940.929135:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575941.248976:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575941.248982:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575941.248985:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575941.248991:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575941.248996:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575941.249001:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575941.249013:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575941.249016:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575941.249019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575941.953077:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575941.953081:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575941.953085:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575942.272956:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575942.272962:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575942.272966:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575942.272970:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575942.272975:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575942.272978:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575942.272994:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575942.272996:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575942.272999:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575942.529097:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311302381, at 4311302400 00000100:00000010:0.0:1788575942.529104:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfe40. 00000020:00000001:0.0:1788575942.977757:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575942.977762:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575942.977766:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575943.296951:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575943.296961:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575943.296966:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575943.296974:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575943.296982:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575943.296988:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575943.297013:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575943.297016:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575943.297019:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575943.808960:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575943.808976:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575943.808981:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575943.808986:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575943.808992:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575943.809012:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16631 00000400:00000200:0.0:1788575943.809022:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575943.809029:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575943.809032:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575943.809034:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809037:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575943.809039:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809041:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575943.809045:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575943.809047:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575943.809049:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575943.809052:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575943.809054:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575943.809056:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575943.809058:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809060:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575943.809083:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809085:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575943.809087:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575943.809091:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809093:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575943.809096:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575943.809100:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575943.809102:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.809104:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575943.809106:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575943.809114:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788575943.809118:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575943.809122:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575943.809124:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.809126:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575943.809129:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575943.809134:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575943.809141:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575943.809148:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575943.809157:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516885632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575943.809166:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575943.809171:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575943.809176:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516885632/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575943.809201:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575943.809205:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575943.809208:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575943.809210:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575943.809211:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575943.809214:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575943.809222:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16631 00000400:00000200:0.0:1788575943.809227:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575943.809230:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575943.809232:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575943.809235:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809239:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575943.809242:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809247:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575943.809252:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575943.809254:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575943.809256:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575943.809261:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575943.809265:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575943.809270:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575943.809278:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809282:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575943.809286:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809290:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575943.809293:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575943.809299:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809304:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575943.809308:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575943.809317:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575943.809322:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.809325:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575943.809332:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575943.809336:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.809338:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575943.809343:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788575943.809346:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575943.809348:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575943.809357:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.809359:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575943.809361:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575943.809364:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575943.809371:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575943.809379:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575943.809385:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516885760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575943.809395:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575943.809399:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575943.809406:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516885760/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575943.809421:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575943.809426:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575943.809432:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575943.809434:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575943.809437:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575943.809439:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575943.809442:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16631 00000400:00000200:0.0:1788575943.809446:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575943.809463:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575943.809467:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575943.809471:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809476:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575943.809479:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809484:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575943.809487:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575943.809490:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575943.809494:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575943.809496:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575943.809499:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575943.809508:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575943.809515:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809520:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575943.809522:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809526:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575943.809531:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575943.809536:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809541:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575943.809545:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575943.809550:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575943.809556:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.809561:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575943.809564:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575943.809569:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.809573:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575943.809578:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788575943.809582:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575943.809587:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575943.809588:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.809590:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575943.809592:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575943.809594:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575943.809600:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575943.809606:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575943.809616:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516885888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575943.809627:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575943.809631:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575943.809639:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516885888/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575943.809655:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575943.809668:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575943.809675:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575943.809678:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575943.809680:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575943.809683:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575943.809689:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16631 00000400:00000200:0.0:1788575943.809695:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575943.809702:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575943.809706:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575943.809711:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809714:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575943.809719:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809725:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575943.809728:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575943.809730:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575943.809733:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575943.809738:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575943.809743:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575943.809747:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575943.809755:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809759:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575943.809776:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809779:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575943.809785:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575943.809790:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575943.809792:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575943.809794:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575943.809798:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575943.809803:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.809807:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575943.809871:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575943.809877:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.809881:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575943.809887:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788575943.809892:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575943.809897:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575943.809899:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.809903:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575943.809905:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575943.809908:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575943.809914:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575943.809921:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575943.809931:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516886016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575943.809947:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575943.809951:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575943.809959:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516886016/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575943.809969:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575943.809975:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575943.809983:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575943.809987:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575943.809989:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575943.809995:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575943.810001:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16631 00000400:00000200:0.0:1788575943.810008:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575943.810015:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575943.810021:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575943.810028:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.810033:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575943.810037:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.810049:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575943.810054:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575943.810057:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575943.810060:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575943.810065:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575943.810068:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575943.810070:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575943.810074:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.810079:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575943.810085:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575943.810091:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575943.810096:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575943.810103:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575943.810108:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575943.810114:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575943.810118:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575943.810123:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.810125:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575943.810128:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575943.810130:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.810131:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575943.810137:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788575943.810141:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575943.810144:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575943.810147:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575943.810151:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575943.810154:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575943.810157:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575943.810164:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788575943.810171:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575943.810181:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516886144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575943.810216:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575943.810220:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575943.810228:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516886144/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575943.810242:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575943.810249:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16641) 00000100:00000001:0.0:1788575944.320946:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575944.320956:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575944.320960:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575944.320969:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575944.320976:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575944.320981:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575944.320993:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575944.320997:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575944.321001:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575945.025142:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575945.025146:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575945.025149:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575945.344958:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575945.344967:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575945.344971:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575945.344979:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575945.344984:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575945.344990:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575945.344995:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575945.345000:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575945.345003:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575945.345075:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575945.345078:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575945.345080:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575945.345082:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575945.345085:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575945.345087:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575945.345466:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575945.345492:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575945.345496:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575946.049134:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575946.049137:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575946.049140:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575946.369404:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575946.369407:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575946.369408:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575946.369411:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575946.369413:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575946.369415:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575946.369417:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575946.369419:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575946.369421:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575947.073029:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575947.073032:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575947.073035:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575947.393095:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575947.393098:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575947.393099:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575947.393102:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575947.393104:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575947.393106:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575947.393107:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575947.393109:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575947.393111:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575947.393123:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575947.393128:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575947.393130:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575947.393133:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575947.393136:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575947.393139:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575947.393146:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575947.393152:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575947.393160:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575947.648935:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311307502, at 4311307520 00000100:00000010:0.0:1788575947.648954:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfba0. 00000020:00000001:0.0:1788575948.096995:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575948.096998:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575948.097001:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575948.417014:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575948.417020:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575948.417024:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575948.417031:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575948.417037:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575948.417040:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575948.417043:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575948.417044:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575948.417047:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575949.440932:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575949.440937:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575949.440941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575949.440946:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575949.440951:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575949.440955:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575949.440971:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575949.440974:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575949.440976:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575950.145095:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575950.145099:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575950.145103:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575950.465483:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575950.465487:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575950.465489:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575950.465494:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575950.465499:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575950.465502:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575950.465506:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575950.465510:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575950.465514:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575950.465547:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575950.465551:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575950.465554:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575950.465556:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575950.465560:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575950.465569:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575950.465571:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575950.465572:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575950.465574:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575951.169050:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575951.169054:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575951.169057:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575951.488940:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575951.488946:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575951.488951:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575951.488959:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575951.488967:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575951.488973:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575951.488978:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575951.488982:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575951.488989:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575952.193002:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575952.193005:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575952.193008:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575952.512949:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575952.512955:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575952.512958:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575952.512962:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575952.512968:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575952.512971:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575952.512976:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575952.512978:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575952.512982:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575952.513007:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575952.513016:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575952.513025:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575952.513034:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575952.513038:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575952.513045:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575952.513049:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575952.513052:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575952.513056:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575952.768926:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311312621, at 4311312640 00000100:00000010:0.0:1788575952.768958:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfc00. 00000020:00000001:0.0:1788575953.217065:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575953.217068:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575953.217071:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575953.536931:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575953.536936:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575953.536944:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575953.536949:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575953.536954:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575953.536958:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575953.536979:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575953.536982:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575953.536985:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575954.048954:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788575954.048963:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788575954.048968:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:0.0:1788575954.560965:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575954.560970:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575954.560972:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575954.560978:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575954.560983:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575954.560988:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575954.561004:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575954.561007:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575954.561010:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575954.561282:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575954.561285:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575954.561286:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575954.561289:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575954.561291:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575954.561293:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575954.561294:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575954.561295:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575954.561297:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575955.265228:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575955.265233:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575955.265242:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575955.585240:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575955.585243:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575955.585245:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575955.585248:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575955.585250:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575955.585253:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575955.585256:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575955.585257:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575955.585259:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575956.289387:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575956.289391:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575956.289395:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575956.609087:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575956.609105:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575956.609111:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575956.609121:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575956.609129:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575956.609135:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575956.609140:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575956.609143:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575956.609147:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575957.315090:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575957.315094:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575957.315097:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575957.633345:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575957.633348:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575957.633349:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575957.633352:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575957.633354:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575957.633357:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575957.633359:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575957.633361:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575957.633365:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575957.891454:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311317741, at 4311317762 00000100:00000010:0.0:1788575957.891470:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916747e0. 00000020:00000001:0.0:1788575958.338962:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575958.338996:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575958.339000:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575958.658420:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575958.658423:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575958.658424:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575958.658428:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575958.658430:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575958.658431:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575958.658433:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575958.658434:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575958.658436:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575959.169350:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575959.169364:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575959.169367:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575959.169372:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575959.169377:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575959.169387:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16646 00000400:00000200:0.0:1788575959.169397:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575959.169405:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575959.169408:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575959.169412:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169415:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575959.169417:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169420:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575959.169425:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575959.169427:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575959.169430:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575959.169433:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575959.169435:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575959.169438:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575959.169441:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169443:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575959.169469:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169471:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575959.169474:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575959.169478:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169481:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575959.169484:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575959.169488:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575959.169491:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.169493:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575959.169496:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575959.169504:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788575959.169509:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575959.169513:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575959.169515:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.169518:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575959.169522:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575959.169527:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575959.169537:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575959.169546:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575959.169558:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516887552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575959.169569:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575959.169573:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575959.169578:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516887552/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575959.169598:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575959.169602:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575959.169606:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575959.169608:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575959.169610:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575959.169613:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575959.169622:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16646 00000400:00000200:0.0:1788575959.169629:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575959.169635:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575959.169639:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575959.169645:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169650:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575959.169654:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169659:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575959.169664:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575959.169666:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575959.169670:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575959.169674:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575959.169678:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575959.169683:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575959.169689:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169695:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575959.169700:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169703:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575959.169708:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575959.169715:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169720:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575959.169724:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575959.169732:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575959.169736:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.169739:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575959.169744:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575959.169750:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.169753:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575959.169759:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788575959.169764:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575959.169771:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575959.169781:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.169786:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575959.169788:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575959.169791:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575959.169799:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575959.169806:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575959.169836:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516887680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575959.169847:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575959.169852:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575959.169859:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516887680/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575959.169875:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575959.169880:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575959.169884:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575959.169888:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575959.169891:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575959.169897:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575959.169901:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16646 00000400:00000200:0.0:1788575959.169909:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575959.169916:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575959.169921:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575959.169928:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169934:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575959.169938:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169940:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575959.169943:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575959.169946:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575959.169948:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575959.169952:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575959.169956:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575959.169964:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575959.169971:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169976:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575959.169979:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169982:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575959.169984:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575959.169985:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575959.169987:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575959.169990:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575959.169994:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575959.169997:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.169999:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575959.170000:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575959.170002:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.170006:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575959.170008:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788575959.170011:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575959.170016:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575959.170018:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.170022:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575959.170024:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575959.170026:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575959.170030:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575959.170036:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575959.170042:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516887808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575959.170050:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575959.170052:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575959.170056:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516887808/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575959.170064:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575959.170070:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575959.170073:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575959.170074:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575959.170075:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575959.170077:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575959.170079:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16646 00000400:00000200:0.0:1788575959.170082:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575959.170084:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575959.170086:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575959.170088:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.170089:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575959.170091:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.170092:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575959.170093:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575959.170094:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575959.170095:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575959.170096:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575959.170097:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575959.170098:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575959.170100:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.170101:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575959.170102:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.170104:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575959.170105:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575959.170107:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575959.170108:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575959.170109:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575959.170111:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575959.170112:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.170113:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575959.170118:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575959.170119:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.170120:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575959.170123:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788575959.170124:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575959.170126:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575959.170127:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.170128:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575959.170129:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575959.170130:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575959.170133:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788575959.170135:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575959.170138:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516887936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575959.170144:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575959.170145:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575959.170149:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516887936/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575959.170156:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575959.170159:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575959.170163:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575959.170165:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575959.170166:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575959.170169:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575959.170171:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16646 00000400:00000200:0.0:1788575959.170175:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575959.170178:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575959.170180:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575959.170183:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.170184:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575959.170186:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.170195:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575959.170198:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575959.170199:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575959.170201:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575959.170203:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575959.170206:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575959.170208:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575959.170213:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.170218:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575959.170223:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575959.170228:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575959.170232:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575959.170239:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575959.170242:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575959.170245:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575959.170250:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575959.170255:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.170260:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575959.170263:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575959.170268:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.170273:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575959.170277:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788575959.170280:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575959.170285:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575959.170286:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575959.170287:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575959.170288:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575959.170290:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575959.170295:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575959.170299:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575959.170304:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516888064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575959.170347:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575959.170351:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575959.170359:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516888064/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575959.170373:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575959.170379:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16656) 00000100:00000001:0.0:1788575959.681506:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575959.681511:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575959.681514:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575959.681517:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575959.681520:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575959.681523:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575959.684971:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575959.684981:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575959.684987:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575960.385219:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575960.385223:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575960.385226:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575960.705603:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575960.705607:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575960.705608:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575960.705611:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575960.705613:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575960.705616:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575960.707400:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575960.707403:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575960.707406:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575961.409101:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575961.409106:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575961.409109:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575961.729083:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575961.729094:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575961.729099:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575961.729108:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575961.729126:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575961.729129:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575961.729140:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575961.729144:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575961.729147:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575962.433231:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575962.433234:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575962.433237:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575962.752995:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575962.753000:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575962.753006:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575962.753012:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575962.753017:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575962.753030:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575962.753085:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575962.753088:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575962.753094:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575963.008929:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311322862, at 4311322880 00000100:00000010:0.0:1788575963.008946:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674c00. 00000020:00000001:0.0:1788575963.457126:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575963.457133:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575963.457138:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575963.776928:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575963.776937:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575963.776941:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575963.776948:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575963.776956:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575963.776963:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575963.776985:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575963.776989:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575963.776992:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575965.505020:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575965.505029:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575965.505032:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575965.825189:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575965.825192:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575965.825193:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575965.825201:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575965.825203:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575965.825204:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575965.825206:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575965.825207:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575965.825209:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575966.529116:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575966.529120:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575966.529122:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575966.849038:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575966.849041:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575966.849042:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575966.849044:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575966.849046:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575966.849047:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575966.849049:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575966.849050:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575966.849051:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575967.553045:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575967.553050:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575967.553053:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575967.872919:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575967.872923:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575967.872927:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575967.872933:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575967.872937:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575967.872941:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575967.872944:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575967.872946:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575967.872949:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575968.128920:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311327983, at 4311328000 00000100:00000010:0.0:1788575968.128936:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674b40. 00000020:00000001:0.0:1788575968.577176:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575968.577184:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575968.577187:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575968.896923:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575968.896929:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575968.896956:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575968.896960:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575968.896963:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575968.896966:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575968.896968:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575968.896970:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575968.896973:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575969.920921:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575969.920930:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575969.920941:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575969.920946:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575969.920950:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575969.920954:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575969.920960:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575969.920964:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575969.920967:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575970.626160:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575970.626163:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575970.626166:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575970.945055:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575970.945061:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575970.945065:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575970.945070:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575970.945074:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575970.945079:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575970.945085:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575970.945087:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575970.945090:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575971.649047:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575971.649051:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575971.649054:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575971.968962:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575971.968969:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575971.968974:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575971.968981:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575971.968988:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575971.969001:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575971.969009:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575971.969028:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575971.969031:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575972.673080:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575972.673084:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575972.673087:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575972.992979:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575972.992984:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575972.992989:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575972.992995:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575972.993000:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575972.993003:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575972.993007:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575972.993010:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575972.993015:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575973.248914:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311333103, at 4311333120 00000100:00000010:0.0:1788575973.248930:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916748a0. 00000020:00000001:0.0:1788575973.696983:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575973.696985:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575973.696988:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575974.016923:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575974.016928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575974.016933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575974.016945:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575974.016949:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575974.016954:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575974.016959:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575974.016963:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575974.016967:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575975.040914:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575975.040923:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575975.040927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575975.040934:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575975.040939:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575975.040950:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575975.040955:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575975.040958:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575975.040968:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575975.041019:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575975.041024:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575975.041027:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575975.041031:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575975.041043:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575975.041046:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575975.041060:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575975.041065:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575975.041069:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575975.744974:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575975.744978:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575975.744980:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575976.064946:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575976.064952:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575976.064956:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575976.064967:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575976.064975:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575976.064980:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575976.064987:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575976.064991:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575976.064994:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575976.769092:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575976.769097:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575976.769101:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575977.089139:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575977.089144:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575977.089146:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575977.089149:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575977.089152:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575977.089154:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575977.089157:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575977.089159:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575977.089161:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575977.793101:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575977.793104:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575977.793107:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575978.112982:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575978.112996:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575978.113027:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575978.113035:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575978.113044:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575978.113049:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575978.113072:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575978.113077:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575978.113084:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575978.368983:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311338221, at 4311338240 00000100:00000010:0.0:1788575978.369014:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf0c0. 00000020:00000001:0.0:1788575978.816999:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575978.817002:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575978.817006:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575979.136938:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575979.136947:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575979.136949:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575979.136955:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575979.136961:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575979.136966:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575979.136969:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575979.136972:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575979.136975:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575979.137545:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575979.137549:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575979.137550:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575979.137553:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575979.137557:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575979.137560:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575979.137562:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575979.137563:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575979.137566:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575980.160952:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575980.160958:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575980.160960:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575980.160965:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575980.160969:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575980.160973:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575980.160981:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575980.161003:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575980.161008:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575980.161454:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575980.161457:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575980.161458:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575980.161461:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575980.161466:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575980.161468:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575980.161470:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575980.161472:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575980.161474:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575980.865063:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575980.865067:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575980.865070:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575981.185094:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575981.185097:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575981.185098:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575981.185101:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575981.185104:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575981.185108:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575981.185255:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575981.185257:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575981.185259:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575981.889450:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575981.889454:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575981.889457:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788575982.913036:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575982.913039:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575982.913042:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788575983.488886:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311343341, at 4311343360 00000100:00000010:0.0:1788575983.488896:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf000. 00000020:00000001:0.0:1788575983.936986:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575983.936989:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575983.936992:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575985.280988:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575985.280994:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575985.280996:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575985.281015:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575985.281018:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575985.281021:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575985.281023:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575985.281025:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575985.281027:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575985.985014:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575985.985017:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575985.985019:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575986.304964:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575986.304972:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575986.304974:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575986.304982:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575986.304986:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575986.304989:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575986.304991:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575986.304993:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575986.304996:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575986.307572:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575986.307576:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575986.307577:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575986.307580:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575986.307584:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575986.307586:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575986.307588:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575986.307590:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575986.307592:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575987.009012:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575987.009016:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575987.009019:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575987.328940:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575987.328946:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575987.328950:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575987.328956:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575987.328962:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575987.328967:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575987.328973:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575987.328976:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575987.328997:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575988.032991:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575988.032993:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575988.032996:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575988.352919:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575988.352927:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575988.352931:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575988.352939:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788575988.352947:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575988.352953:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575988.352968:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575988.352973:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575988.352977:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575988.353010:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575988.353012:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575988.353013:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575988.353015:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575988.353017:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575988.353019:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575988.353021:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575988.353022:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575988.353025:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575988.608914:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311348461, at 4311348480 00000100:00000010:0.0:1788575988.608927:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf060. 00000020:00000001:0.0:1788575989.056990:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575989.056993:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575989.056995:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575989.376944:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575989.376952:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575989.376956:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575989.376964:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575989.376971:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575989.376977:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575989.376993:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575989.376997:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575989.377002:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575990.400904:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575990.400922:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575990.400924:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575990.400931:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575990.400937:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575990.400942:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575990.400956:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575990.400960:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575990.400965:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575991.105020:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575991.105023:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575991.105025:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575991.424944:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575991.424948:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575991.424953:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575991.424958:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575991.424964:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575991.424968:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575991.424982:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575991.424985:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575991.424989:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575992.129037:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575992.129042:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575992.129046:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575992.448910:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575992.448919:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575992.448923:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575992.448930:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575992.448937:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575992.448943:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575992.448958:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575992.448962:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575992.448965:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788575992.960926:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788575992.960933:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788575992.960937:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788575992.961098:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575992.961111:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788575992.961127:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788575992.961130:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788575993.153015:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575993.153019:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575993.153023:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575993.473004:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575993.473009:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575993.473013:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575993.473027:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575993.473032:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575993.473036:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575993.473040:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575993.473043:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575993.473046:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575993.728917:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311353581, at 4311353600 00000100:00000010:0.0:1788575993.728932:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf240. 00000020:00000001:0.0:1788575994.176995:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575994.176998:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575994.177000:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575994.496873:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575994.496880:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575994.496883:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575994.496889:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788575994.496894:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575994.496897:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575994.496906:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575994.496908:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575994.496912:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575995.008945:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575995.008956:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788575995.008958:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575995.008962:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575995.008966:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575995.008972:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16682 00000400:00000200:0.0:1788575995.008980:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575995.009000:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575995.009003:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575995.009006:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009008:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575995.009011:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009013:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575995.009016:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575995.009018:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788575995.009021:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575995.009023:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788575995.009026:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788575995.009028:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788575995.009030:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009032:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575995.009034:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009036:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575995.009038:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788575995.009041:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009044:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575995.009047:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575995.009050:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575995.009053:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.009055:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788575995.009057:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575995.009063:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788575995.009066:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788575995.009070:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575995.009071:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.009074:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575995.009076:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575995.009079:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575995.009086:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788575995.009098:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575995.009108:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516892032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575995.009117:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575995.009122:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575995.009126:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516892032/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575995.009144:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575995.009146:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575995.009149:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788575995.009151:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575995.009152:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575995.009155:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575995.009157:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16682 00000400:00000200:0.0:1788575995.009161:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575995.009164:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575995.009167:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575995.009169:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009172:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575995.009178:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009182:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575995.009187:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575995.009189:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788575995.009193:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575995.009195:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788575995.009196:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788575995.009202:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788575995.009210:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009214:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575995.009226:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009229:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575995.009236:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788575995.009242:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009248:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575995.009252:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575995.009259:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575995.009265:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.009267:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575995.009271:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575995.009275:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.009278:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575995.009284:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788575995.009288:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575995.009294:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575995.009305:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.009309:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575995.009312:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575995.009315:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575995.009322:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788575995.009327:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575995.009334:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516892160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575995.009349:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575995.009359:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575995.009369:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516892160/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575995.009380:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575995.009384:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575995.009390:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788575995.009394:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575995.009402:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575995.009407:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575995.009412:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16682 00000400:00000200:0.0:1788575995.009422:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575995.009431:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575995.009437:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575995.009449:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009454:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575995.009461:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009465:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575995.009472:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575995.009475:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788575995.009479:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575995.009481:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788575995.009483:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788575995.009487:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788575995.009496:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009501:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575995.009508:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009522:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575995.009526:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788575995.009534:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009539:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575995.009541:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575995.009546:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575995.009551:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.009555:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575995.009565:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788575995.009569:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.009572:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575995.009582:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788575995.009586:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788575995.009592:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575995.009594:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.009596:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575995.009599:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575995.009601:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575995.009606:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788575995.009612:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575995.009622:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516892288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575995.009644:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575995.009650:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575995.009662:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516892288/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575995.009681:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575995.009688:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575995.009696:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788575995.009701:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575995.009705:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575995.009712:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575995.009719:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16682 00000400:00000200:0.0:1788575995.009727:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575995.009736:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575995.009741:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575995.009748:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009753:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575995.009758:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009762:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575995.009767:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575995.009775:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788575995.009780:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575995.009785:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788575995.009789:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788575995.009825:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788575995.009830:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009832:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575995.009834:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009835:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575995.009838:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788575995.009840:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575995.009843:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575995.009849:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575995.009852:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575995.009856:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.009860:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575995.009865:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575995.009871:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.009875:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575995.009882:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788575995.009887:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575995.009894:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575995.009898:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.009903:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575995.009907:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575995.009911:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575995.009919:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788575995.009927:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575995.009937:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516892416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575995.009952:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575995.009956:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575995.009969:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516892416/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575995.009987:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788575995.009993:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788575995.010001:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788575995.010005:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788575995.010009:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788575995.010014:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788575995.010021:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16682 00000400:00000200:0.0:1788575995.010030:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788575995.010039:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788575995.010045:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788575995.010053:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.010059:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575995.010065:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.010071:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788575995.010077:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788575995.010082:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788575995.010087:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788575995.010092:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788575995.010098:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788575995.010103:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788575995.010108:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.010112:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788575995.010118:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788575995.010122:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788575995.010129:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788575995.010136:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788575995.010143:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788575995.010153:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788575995.010159:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788575995.010165:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.010171:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788575995.010176:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788575995.010182:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.010187:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788575995.010193:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788575995.010200:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788575995.010207:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788575995.010210:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575995.010215:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788575995.010219:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788575995.010224:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788575995.010233:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788575995.010241:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575995.010252:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516892544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788575995.010272:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788575995.010277:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575995.010286:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516892544/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788575995.010310:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575995.010317:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16692) 00000100:00000001:0.0:1788575995.521033:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575995.521037:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575995.521038:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575995.521040:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575995.521041:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575995.521042:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575995.521326:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575995.521327:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575995.521329:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575996.224983:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575996.225000:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575996.225003:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575996.544922:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575996.544931:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575996.544936:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575996.544944:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575996.544951:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575996.544957:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575996.544974:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575996.544976:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575996.544978:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575996.545033:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575996.545038:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575996.545041:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575996.545045:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575996.545049:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575996.545052:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575996.545057:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575996.545058:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575996.545060:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575997.249007:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575997.249011:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575997.249015:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575997.568934:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575997.568943:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575997.568947:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575997.568957:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788575997.568965:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575997.568970:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575997.568985:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575997.568988:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575997.568991:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788575997.569007:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575997.569012:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575997.569013:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575997.569016:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788575997.569019:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575997.569022:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575997.569031:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575997.569033:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575997.569035:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788575998.273346:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575998.273356:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575998.273359:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575998.593111:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575998.593114:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575998.593114:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575998.593117:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575998.593118:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575998.593120:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575998.593121:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575998.593122:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575998.593124:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788575998.851267:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311358701, at 4311358722 00000100:00000010:0.0:1788575998.851276:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf000. 00000020:00000001:0.0:1788575999.297051:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788575999.297055:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788575999.297057:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788575999.616930:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788575999.616950:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788575999.616954:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788575999.616961:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788575999.616965:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788575999.616971:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575999.616975:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788575999.616979:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788575999.616985:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576000.128961:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576000.128971:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788576000.128974:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576000.128977:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576000.128982:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576000.128988:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16687 00000400:00000200:0.0:1788576000.128997:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576000.129017:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576000.129020:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576000.129023:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129026:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576000.129028:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129031:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576000.129035:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576000.129036:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788576000.129039:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576000.129041:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788576000.129044:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788576000.129046:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788576000.129049:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129051:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576000.129053:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129055:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576000.129057:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788576000.129060:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129062:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576000.129065:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576000.129067:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576000.129070:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.129072:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788576000.129074:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576000.129080:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788576000.129083:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788576000.129086:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576000.129087:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.129090:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576000.129093:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576000.129096:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576000.129104:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788576000.129115:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576000.129123:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516892672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576000.129133:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576000.129137:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576000.129142:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516892672/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576000.129161:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576000.129164:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576000.129168:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788576000.129169:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576000.129171:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576000.129173:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576000.129175:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16687 00000400:00000200:0.0:1788576000.129180:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576000.129184:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576000.129187:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576000.129191:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129194:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576000.129199:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129203:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576000.129206:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576000.129209:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788576000.129212:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576000.129215:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788576000.129219:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788576000.129222:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788576000.129225:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129227:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576000.129236:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129240:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576000.129244:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788576000.129250:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129255:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576000.129259:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576000.129264:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576000.129268:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.129271:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576000.129277:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576000.129281:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.129284:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576000.129289:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788576000.129292:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576000.129297:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576000.129300:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.129303:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576000.129305:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576000.129307:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576000.129313:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788576000.129320:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576000.129328:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516892800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576000.129346:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576000.129359:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576000.129366:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516892800/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576000.129380:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576000.129384:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576000.129390:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788576000.129393:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576000.129401:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576000.129407:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576000.129414:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16687 00000400:00000200:0.0:1788576000.129422:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576000.129426:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576000.129428:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576000.129431:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129436:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576000.129442:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129446:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576000.129450:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576000.129453:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788576000.129455:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576000.129457:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788576000.129461:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788576000.129466:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788576000.129472:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129477:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576000.129483:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129487:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576000.129492:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788576000.129499:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129503:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576000.129515:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576000.129523:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576000.129528:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.129532:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576000.129536:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576000.129540:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.129544:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576000.129557:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788576000.129563:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576000.129568:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576000.129570:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.129572:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576000.129575:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576000.129578:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576000.129584:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788576000.129589:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576000.129597:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516892928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576000.129614:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576000.129619:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576000.129625:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516892928/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576000.129643:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576000.129648:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576000.129652:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788576000.129655:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576000.129658:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576000.129661:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576000.129666:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16687 00000400:00000200:0.0:1788576000.129672:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576000.129678:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576000.129682:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576000.129687:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129691:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576000.129696:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129700:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576000.129704:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576000.129714:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788576000.129721:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576000.129725:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788576000.129730:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788576000.129736:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788576000.129743:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129747:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576000.129753:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129755:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576000.129761:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788576000.129767:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129772:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576000.129775:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576000.129779:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576000.129784:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.129787:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576000.129789:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576000.129794:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.129798:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576000.129804:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788576000.129832:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576000.129837:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576000.129840:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.129845:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576000.129848:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576000.129851:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576000.129857:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788576000.129862:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576000.129871:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516893056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576000.129884:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576000.129886:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576000.129897:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516893056/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576000.129911:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576000.129915:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576000.129919:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788576000.129920:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576000.129922:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576000.129924:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576000.129927:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16687 00000400:00000200:0.0:1788576000.129932:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576000.129937:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576000.129942:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576000.129960:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129966:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576000.129970:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.129975:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576000.129980:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576000.129983:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788576000.129986:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576000.129988:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788576000.129992:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788576000.129997:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788576000.130004:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.130007:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576000.130011:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576000.130014:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576000.130016:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788576000.130019:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576000.130021:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576000.130029:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576000.130034:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576000.130038:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.130043:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576000.130047:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576000.130051:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.130054:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576000.130058:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788576000.130062:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576000.130066:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576000.130069:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.130074:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576000.130076:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576000.130078:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576000.130083:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788576000.130087:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576000.130095:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516893184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576000.130109:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576000.130113:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576000.130120:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516893184/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576000.130132:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576000.130137:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16697) 00000100:00000001:0.0:1788576000.640893:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576000.640903:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576000.640906:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576000.640914:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576000.640920:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576000.640926:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576000.640932:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576000.640936:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576000.640939:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576000.640978:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576000.641002:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576000.641008:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576000.641012:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576000.641015:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576000.641019:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576000.641023:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576000.641027:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576000.641031:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576001.345076:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576001.345080:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576001.345082:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576001.664936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576001.664945:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576001.664949:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576001.664956:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576001.664969:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576001.664973:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576001.665013:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576001.665016:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576001.665021:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576002.369057:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576002.369060:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576002.369062:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576002.688918:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576002.688925:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576002.688927:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576002.688931:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576002.688935:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576002.688938:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576002.688941:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576002.688944:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576002.688950:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576003.392974:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576003.392976:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576003.392979:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576003.712936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576003.712945:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576003.712949:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576003.712961:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576003.712965:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576003.712971:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576003.712990:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576003.712994:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576003.712997:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576003.968918:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311363821, at 4311363840 00000100:00000010:0.0:1788576003.968935:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf660. 00000020:00000001:0.0:1788576004.417022:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576004.417027:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576004.417030:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576004.736972:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576004.736979:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576004.736983:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576004.736989:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576004.736994:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576004.736999:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576004.737003:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576004.737007:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576004.737012:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576005.760932:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576005.760941:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576005.760945:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576005.760953:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576005.760960:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576005.760967:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576005.760980:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576005.760983:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576005.760987:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576006.465006:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576006.465010:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576006.465013:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576006.784945:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576006.784952:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576006.784957:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576006.784965:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576006.784969:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576006.784990:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576006.784992:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576006.784994:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576006.784998:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576007.488984:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576007.488988:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576007.488990:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576007.808964:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576007.808970:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576007.808972:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576007.808977:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576007.808981:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576007.808985:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576007.808988:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576007.809000:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576007.809004:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576007.809514:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576007.809517:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576007.809518:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576007.809521:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576007.809524:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576007.809526:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576007.809528:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576007.809529:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576007.809531:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576008.513105:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576008.513109:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576008.513112:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576008.835428:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576008.835436:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576008.835440:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576008.835445:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576008.835451:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576008.835457:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576008.836269:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576008.836273:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576008.836275:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576009.088948:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311368941, at 4311368960 00000100:00000010:0.0:1788576009.088982:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf7e0. 00000020:00000001:0.0:1788576009.537075:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576009.537078:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576009.537080:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576009.856928:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576009.856937:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576009.856940:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576009.856945:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576009.856948:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576009.856953:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576009.856956:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576009.856958:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576009.856961:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576009.857091:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576009.857094:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576009.857096:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576009.857098:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576009.857100:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576009.857102:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576009.859891:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576009.859896:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576009.859899:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576010.880936:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576010.880957:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576010.880962:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576010.880966:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576010.880972:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576010.880977:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576010.880981:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576010.880985:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576010.880989:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576011.586888:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576011.586893:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576011.586895:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576011.904995:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576011.904999:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576011.905000:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576011.905024:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576011.905027:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576011.905032:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576011.905035:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576011.905037:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576011.905039:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576012.609523:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576012.609527:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576012.609531:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576012.928934:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576012.928944:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576012.928955:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576012.928961:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576012.928965:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576012.928970:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576012.928976:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576012.928981:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576012.928984:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576012.929351:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576012.929356:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576012.929360:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576012.929365:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576012.929371:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576012.929375:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576012.929377:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576012.929378:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576012.929381:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576013.633408:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576013.633411:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576013.633415:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576013.952907:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576013.952917:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576013.952919:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576013.952924:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576013.952928:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576013.952932:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576013.952936:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576013.952938:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576013.952950:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576014.208930:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311374061, at 4311374080 00000100:00000010:0.0:1788576014.208945:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfc00. 00000020:00000001:0.0:1788576014.658781:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576014.658788:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576014.658793:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576014.977095:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576014.977104:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576014.977107:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576014.977113:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576014.977117:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576014.977121:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576014.977126:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576014.977130:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576014.977152:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576014.977433:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576014.977436:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576014.977437:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576014.977440:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576014.977442:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576014.977444:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576014.977446:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576014.977447:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576014.977449:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576016.000948:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576016.000957:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576016.000961:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576016.000971:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576016.000977:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576016.000984:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576016.001004:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576016.001007:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576016.001013:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576016.705015:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576016.705019:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576016.705022:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576017.025139:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576017.025146:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576017.025147:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576017.025149:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576017.025150:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576017.025151:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576017.025152:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576017.025154:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576017.025155:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576017.729054:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576017.729057:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576017.729060:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576018.048973:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576018.048984:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576018.048986:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576018.048993:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576018.048998:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576018.049003:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576018.049007:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576018.049010:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576018.049014:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576018.049035:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576018.049041:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576018.049044:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576018.049049:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576018.049055:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576018.049061:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576018.049068:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576018.049072:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576018.049079:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576018.755869:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576018.755872:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576018.755875:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576019.072984:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576019.072997:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576019.073001:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576019.073009:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576019.073015:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576019.073020:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576019.073048:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576019.073051:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576019.073056:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576019.328950:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311379181, at 4311379200 00000100:00000010:0.0:1788576019.328970:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf300. 00000020:00000001:0.0:1788576019.777103:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576019.777106:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576019.777109:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576020.096964:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576020.096974:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576020.096977:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576020.096985:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576020.096989:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576020.096994:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576020.096997:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576020.096999:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576020.097161:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576021.120964:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576021.120977:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576021.120982:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576021.120993:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576021.121000:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576021.121008:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576021.122926:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576021.122934:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576021.122940:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576021.825122:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576021.825127:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576021.825131:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576022.145035:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576022.145045:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576022.145048:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576022.145056:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576022.145061:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576022.145065:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576022.145069:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576022.145073:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576022.145094:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576022.849065:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576022.849070:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576022.849073:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576023.169159:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576023.169163:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576023.169164:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576023.169166:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576023.169168:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576023.169169:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576023.169170:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576023.169172:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576023.169173:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576023.873141:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576023.873149:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576023.873153:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576024.192963:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576024.192971:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576024.192974:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576024.192980:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576024.192987:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576024.192991:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576024.192994:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576024.192997:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576024.193001:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576024.193076:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576024.193080:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576024.193081:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576024.193084:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576024.193087:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576024.193090:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576024.193093:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576024.193095:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576024.193101:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576024.448943:0:265949:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311384301, at 4311384320 00000100:00000010:0.0:1788576024.448962:0:265949:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfa20. 00000020:00000001:0.0:1788576024.897062:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576024.897082:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576024.897087:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576025.217127:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576025.217132:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576025.217133:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576025.217136:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576025.217225:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576025.217228:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576025.218016:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576025.218020:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576025.218022:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576026.241049:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576026.241056:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576026.241059:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576026.241064:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576026.241069:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576026.241073:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576026.241770:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576026.241774:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576026.241778:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576026.945023:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576026.945027:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576026.945031:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576027.264972:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576027.264982:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576027.264985:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576027.264994:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576027.265002:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576027.265008:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576027.265022:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576027.265027:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576027.265034:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576027.969164:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576027.969168:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576027.969171:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576028.288941:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576028.288950:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576028.288954:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576028.288976:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576028.288982:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576028.288986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576028.288990:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576028.288994:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576028.288999:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576028.993124:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576028.993126:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576028.993128:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576029.313150:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576029.313155:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576029.313159:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576029.313164:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576029.313169:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576029.313173:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576029.313176:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576029.313177:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576029.313179:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576029.568931:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311389422, at 4311389440 00000100:00000010:0.0:1788576029.568948:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfcc0. 00000020:00000001:0.0:1788576030.017113:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576030.017116:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576030.017119:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576030.337173:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576030.337177:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576030.337178:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576030.337181:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576030.337183:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576030.337185:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576030.337187:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576030.337196:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576030.337199:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788576030.848957:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788576030.848968:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788576030.848973:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1788576030.849121:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576030.849147:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788576030.849151:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576030.849154:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576031.361056:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576031.361061:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576031.361063:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576031.361067:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576031.361070:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576031.361073:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576031.361075:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576031.361077:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576031.361079:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576031.361333:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576031.361336:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576031.361337:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576031.361339:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576031.361341:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576031.361343:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576031.361344:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576031.361346:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576031.361347:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576032.065324:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576032.065328:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576032.065332:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576032.384990:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576032.385010:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576032.385015:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576032.385028:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576032.385036:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576032.385045:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576032.385177:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576032.385185:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576032.385191:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576033.089157:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576033.089162:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576033.089166:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576033.409014:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576033.409024:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576033.409049:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576033.409055:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576033.409062:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576033.409066:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576033.409070:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576033.409073:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576033.409077:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576033.412866:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576033.412887:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576033.412889:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576033.412896:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576033.412899:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576033.412903:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576033.412906:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576033.412908:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576033.412911:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576034.113286:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576034.113290:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576034.113294:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576034.432972:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576034.432975:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576034.432976:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576034.432979:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576034.432981:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576034.432998:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576034.433308:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576034.433312:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576034.433318:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576034.688914:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311394541, at 4311394560 00000100:00000010:0.0:1788576034.688925:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674600. 00000020:00000001:0.0:1788576035.137201:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576035.137210:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576035.137215:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576035.456905:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576035.456914:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576035.456919:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576035.456925:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576035.456955:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576035.456962:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576035.456970:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576035.456980:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576035.456999:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576035.457061:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576035.457065:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576035.457069:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576035.457073:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576035.457076:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576035.457080:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576035.457083:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576035.457085:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576035.457088:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576036.480939:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576036.480946:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576036.480948:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576036.480953:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576036.480956:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576036.480959:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576036.480962:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576036.480964:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576036.480967:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576036.480993:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576036.481001:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576036.481008:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576036.481012:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576036.481016:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576036.481019:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576036.481026:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576036.481029:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576036.481034:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576036.481118:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576036.481123:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576036.481126:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576036.481131:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576036.481138:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576036.481141:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576036.481158:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576036.481160:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576036.481163:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576037.185110:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576037.185115:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576037.185119:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576037.504984:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576037.504996:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576037.505002:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576037.505012:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576037.505018:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576037.505024:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576037.505037:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576037.505041:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576037.505045:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576038.209081:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576038.209084:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576038.209087:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576038.529010:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576038.529028:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576038.529035:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576038.529044:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576038.529051:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576038.529058:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576038.529066:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576038.529071:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576038.529075:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576038.529176:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576038.529181:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576038.529184:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576038.529189:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576038.529195:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576038.529200:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576038.529205:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576038.529208:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576038.529213:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576039.233548:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576039.233556:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576039.233577:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576039.552959:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576039.552968:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576039.552973:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576039.552983:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576039.552990:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576039.553000:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576039.553013:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576039.553017:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576039.553020:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576039.553038:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576039.553041:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576039.553043:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576039.553045:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576039.553047:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576039.553050:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576039.553052:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576039.553053:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576039.553055:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576039.809001:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311399662, at 4311399680 00000100:00000010:0.0:1788576039.809035:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf360. 00000020:00000001:0.0:1788576040.257626:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576040.257633:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576040.257638:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576040.578742:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576040.578746:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576040.578748:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576040.578751:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576040.578753:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576040.578755:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576040.578757:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576040.578759:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576040.578761:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576041.088947:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576041.088961:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788576041.088965:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576041.088994:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576041.089000:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576041.089010:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16728 00000400:00000200:0.0:1788576041.089022:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576041.089030:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576041.089034:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576041.089051:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089055:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576041.089057:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089060:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576041.089065:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576041.089068:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788576041.089071:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576041.089074:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788576041.089077:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788576041.089080:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788576041.089083:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089085:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576041.089087:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089089:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576041.089093:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788576041.089096:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089099:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576041.089102:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576041.089107:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576041.089110:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.089113:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788576041.089115:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576041.089135:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788576041.089140:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788576041.089145:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576041.089152:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.089156:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576041.089159:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576041.089164:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576041.089174:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788576041.089182:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576041.089193:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516897792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576041.089203:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576041.089208:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576041.089214:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516897792/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576041.089235:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576041.089239:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576041.089243:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788576041.089244:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576041.089246:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576041.089250:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576041.089253:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16728 00000400:00000200:0.0:1788576041.089258:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576041.089264:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576041.089268:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576041.089276:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089282:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576041.089288:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089293:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576041.089297:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576041.089300:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788576041.089305:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576041.089311:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788576041.089325:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788576041.089335:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788576041.089345:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089350:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576041.089354:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089358:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576041.089361:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788576041.089368:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089374:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576041.089378:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576041.089387:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576041.089393:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.089397:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576041.089404:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576041.089409:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.089412:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576041.089418:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788576041.089423:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576041.089428:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576041.089431:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.089436:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576041.089440:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576041.089445:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576041.089452:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788576041.089459:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576041.089469:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516897920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576041.089485:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576041.089489:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576041.089497:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516897920/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576041.089521:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576041.089535:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576041.089543:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788576041.089547:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576041.089551:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576041.089558:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576041.089564:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16728 00000400:00000200:0.0:1788576041.089573:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576041.089581:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576041.089599:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576041.089604:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089610:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576041.089615:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089620:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576041.089625:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576041.089627:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788576041.089632:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576041.089635:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788576041.089638:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788576041.089642:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788576041.089647:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089652:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576041.089657:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089662:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576041.089665:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788576041.089673:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089679:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576041.089683:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576041.089687:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576041.089690:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.089692:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576041.089702:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576041.089708:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.089712:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576041.089718:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788576041.089723:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576041.089726:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576041.089729:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.089733:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576041.089737:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576041.089739:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576041.089744:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788576041.089749:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576041.089759:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516898048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576041.089788:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576041.089793:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576041.089802:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516898048/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576041.089837:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576041.089840:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576041.089845:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788576041.089849:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576041.089853:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576041.089859:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576041.089866:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16728 00000400:00000200:0.0:1788576041.089876:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576041.089884:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576041.089890:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576041.089898:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089904:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576041.089911:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089926:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576041.089932:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576041.089936:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788576041.089942:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576041.089946:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788576041.089952:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788576041.089957:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788576041.089967:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089971:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576041.089979:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089984:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576041.089989:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788576041.089995:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576041.089998:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576041.090003:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576041.090009:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576041.090012:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.090016:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576041.090019:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576041.090021:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.090023:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576041.090027:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788576041.090033:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576041.090039:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576041.090042:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.090046:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576041.090049:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576041.090053:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576041.090060:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788576041.090066:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576041.090075:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516898176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576041.090137:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576041.090142:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576041.090152:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516898176/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576041.090173:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576041.090180:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576041.090189:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788576041.090193:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576041.090197:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576041.090204:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576041.090211:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16728 00000400:00000200:0.0:1788576041.090224:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576041.090233:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576041.090239:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576041.090246:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.090251:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576041.090256:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.090259:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576041.090264:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576041.090267:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788576041.090271:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576041.090275:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788576041.090281:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788576041.090287:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788576041.090293:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.090299:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576041.090305:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576041.090310:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576041.090323:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788576041.090330:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576041.090334:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576041.090339:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576041.090343:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576041.090347:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.090352:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576041.090356:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576041.090363:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.090367:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576041.090375:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788576041.090381:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576041.090386:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576041.090390:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.090394:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576041.090397:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576041.090402:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576041.090409:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788576041.090417:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576041.090432:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516898304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576041.090455:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576041.090460:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576041.090472:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516898304/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576041.090483:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576041.090490:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16738) 00000100:00000001:0.0:1788576041.600949:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576041.600959:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576041.600962:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576041.600969:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576041.600976:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576041.601002:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576041.601020:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576041.601023:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576041.601026:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576041.601085:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576041.601089:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576041.601091:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576041.601093:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576041.601096:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576041.601099:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576041.601101:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576041.601103:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576041.601106:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576042.305117:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576042.305122:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576042.305126:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576042.624955:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576042.624965:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576042.624967:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576042.624973:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576042.624978:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576042.624983:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576042.624987:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576042.624990:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576042.624994:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576042.625213:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576042.625216:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576042.625217:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576042.625219:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576042.625220:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576042.625222:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576042.625223:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576042.625225:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576042.625227:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576043.329073:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576043.329076:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576043.329079:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576043.648971:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576043.648996:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576043.649000:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576043.649005:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576043.649015:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576043.649019:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576043.649024:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576043.649028:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576043.649031:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576044.353126:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576044.353148:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576044.353153:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576044.673182:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576044.673187:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576044.673189:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576044.673193:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576044.673196:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576044.673199:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576044.673202:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576044.673204:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576044.673206:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576044.928976:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311404781, at 4311404800 00000100:00000010:0.0:1788576044.929000:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfea0. 00000020:00000001:0.0:1788576045.377123:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576045.377127:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576045.377131:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576045.697237:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576045.697242:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576045.697244:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576045.697247:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576045.697250:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576045.697254:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576045.697256:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576045.697258:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576045.697261:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576046.720976:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576046.720987:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576046.720991:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576046.721030:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576046.721035:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576046.721039:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576046.721046:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576046.721051:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576046.721054:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576047.425184:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576047.425192:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576047.425197:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576047.744925:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576047.744937:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576047.744940:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576047.744947:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576047.744953:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576047.744958:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576047.744966:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576047.744970:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576047.744977:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576047.745014:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576047.745020:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576047.745027:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576047.745043:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576047.745057:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576047.745066:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576047.745077:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576047.745079:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576047.745083:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576048.449465:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576048.449472:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576048.449477:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788576049.473250:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576049.473257:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576049.473262:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788576050.048988:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311409902, at 4311409920 00000100:00000010:0.0:1788576050.049005:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674e40. 00000020:00000001:0.0:1788576050.498013:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576050.498019:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576050.498048:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576050.816961:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576050.816967:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576050.816971:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576050.816982:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576050.816993:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576050.817004:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576050.817036:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576050.817039:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576050.817042:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576051.840982:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576051.840988:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576051.840993:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576051.841000:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576051.841005:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576051.841009:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576051.841020:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576051.841021:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576051.841023:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576052.545026:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576052.545029:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576052.545032:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576052.864924:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576052.864932:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576052.864937:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576052.864944:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576052.864949:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576052.864954:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576052.864963:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576052.864967:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576052.864974:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576052.865016:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576052.865020:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576052.865022:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576052.865025:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576052.865028:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576052.865030:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576052.865042:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576052.865062:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576052.865065:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576053.569004:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576053.569007:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576053.569009:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576053.888942:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576053.888952:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576053.888955:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576053.888971:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576053.888981:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576053.888986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576053.889000:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576053.889006:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576053.889018:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576054.593086:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576054.593091:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576054.593095:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576054.912949:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576054.912955:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576054.912958:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576054.912966:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576054.912973:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576054.912981:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576054.912998:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576054.913001:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576054.913004:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576054.913088:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576054.913093:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576054.913096:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576054.913099:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576054.913104:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576054.913108:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576054.913121:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576054.913124:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576054.913126:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576055.168935:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311415021, at 4311415040 00000100:00000010:0.0:1788576055.168952:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cff60. 00000020:00000001:0.0:1788576055.618768:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576055.618771:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576055.618774:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576055.937021:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576055.937027:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576055.937032:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576055.937045:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576055.937057:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576055.937060:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576055.937069:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576055.937072:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576055.937075:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576056.960930:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576056.960935:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576056.960938:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576056.960957:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576056.960961:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576056.960965:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576056.960970:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576056.960974:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576056.960978:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576056.962308:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576056.962313:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576056.962314:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576056.962318:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576056.962322:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576056.962326:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576056.962330:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576056.962333:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576056.962336:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576057.665097:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576057.665100:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576057.665103:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788576058.689005:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576058.689007:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576058.689010:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576059.009073:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576059.009076:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576059.009095:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576059.009098:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576059.009100:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576059.009101:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576059.009284:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576059.009286:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576059.009288:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576059.713011:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576059.713014:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576059.713018:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576060.032919:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576060.032928:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576060.032933:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576060.032941:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576060.032948:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576060.032953:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576060.032958:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576060.032961:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576060.032965:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576060.288925:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311420142, at 4311420160 00000100:00000010:0.0:1788576060.288949:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674d20. 00000020:00000001:0.0:1788576060.737066:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576060.737068:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576060.737070:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576061.057110:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576061.057113:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576061.057115:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576061.057117:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576061.057119:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576061.057121:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576061.057123:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576061.057125:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576061.057127:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576061.568966:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576061.568982:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788576061.568985:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576061.569014:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576061.569019:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576061.569027:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16749 00000400:00000200:0.0:1788576061.569037:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576061.569042:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576061.569045:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576061.569048:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569050:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576061.569052:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569054:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576061.569058:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576061.569059:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788576061.569061:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576061.569064:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788576061.569065:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788576061.569068:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788576061.569071:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569073:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576061.569075:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569077:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576061.569080:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788576061.569083:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569086:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576061.569090:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576061.569093:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576061.569097:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.569099:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788576061.569101:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576061.569110:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788576061.569114:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788576061.569119:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576061.569129:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.569131:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576061.569134:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576061.569139:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576061.569148:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788576061.569156:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576061.569168:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516900352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576061.569192:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576061.569196:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576061.569200:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516900352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576061.569218:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576061.569221:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576061.569223:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788576061.569225:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576061.569226:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576061.569228:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576061.569229:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16749 00000400:00000200:0.0:1788576061.569233:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576061.569235:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576061.569236:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576061.569238:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569241:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576061.569243:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569246:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576061.569248:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576061.569249:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788576061.569251:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576061.569254:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788576061.569256:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788576061.569264:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788576061.569269:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569273:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576061.569277:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569281:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576061.569285:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788576061.569291:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569297:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576061.569302:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576061.569311:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576061.569315:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.569328:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576061.569334:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576061.569340:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.569344:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576061.569350:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788576061.569355:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576061.569361:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576061.569364:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.569366:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576061.569367:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576061.569370:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576061.569375:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788576061.569390:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576061.569398:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516900480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576061.569407:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576061.569408:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576061.569413:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516900480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576061.569425:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576061.569435:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576061.569442:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788576061.569446:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576061.569448:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576061.569451:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576061.569455:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16749 00000400:00000200:0.0:1788576061.569461:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576061.569466:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576061.569467:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576061.569472:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569477:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576061.569479:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569481:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576061.569483:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576061.569485:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788576061.569488:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576061.569491:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788576061.569492:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788576061.569496:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788576061.569502:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569504:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576061.569511:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569515:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576061.569518:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788576061.569519:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569521:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576061.569523:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576061.569528:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576061.569532:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.569535:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576061.569542:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576061.569547:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.569551:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576061.569557:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788576061.569559:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576061.569565:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576061.569568:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.569571:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576061.569573:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576061.569574:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576061.569580:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788576061.569585:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576061.569591:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516900608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576061.569600:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576061.569601:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576061.569607:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516900608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576061.569615:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576061.569618:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576061.569624:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788576061.569627:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576061.569630:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576061.569633:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576061.569637:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16749 00000400:00000200:0.0:1788576061.569641:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576061.569649:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576061.569654:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576061.569660:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569664:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576061.569671:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569683:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576061.569687:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576061.569691:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788576061.569694:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576061.569698:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788576061.569703:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788576061.569708:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788576061.569714:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569718:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576061.569723:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569729:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576061.569733:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788576061.569738:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569741:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576061.569742:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576061.569745:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576061.569748:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.569750:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576061.569756:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576061.569761:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.569764:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576061.569770:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788576061.569774:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576061.569779:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576061.569781:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.569792:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576061.569796:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576061.569800:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576061.569806:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788576061.569843:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576061.569851:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516900736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576061.569883:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576061.569885:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576061.569888:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516900736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576061.569902:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576061.569907:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576061.569916:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788576061.569920:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576061.569923:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576061.569930:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576061.569937:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16749 00000400:00000200:0.0:1788576061.569945:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576061.569953:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576061.569958:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576061.569966:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569971:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576061.569976:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.569980:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576061.569986:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576061.569989:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788576061.569992:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576061.569997:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788576061.570002:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788576061.570008:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788576061.570015:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.570017:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576061.570019:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576061.570025:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576061.570036:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788576061.570045:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576061.570051:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576061.570054:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576061.570057:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576061.570063:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.570068:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576061.570072:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576061.570076:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.570078:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576061.570085:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788576061.570090:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576061.570096:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576061.570100:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576061.570105:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576061.570107:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576061.570112:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576061.570119:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788576061.570125:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576061.570135:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516900864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576061.570150:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576061.570155:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576061.570163:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516900864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576061.570170:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576061.570187:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16759) 00000100:00000001:0.0:1788576062.080972:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576062.080988:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576062.080992:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576062.081002:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576062.081011:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576062.081049:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576062.081068:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576062.081072:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576062.081075:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576062.786261:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576062.786268:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576062.786273:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576063.104966:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576063.104976:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576063.104980:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576063.104990:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576063.105001:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576063.105005:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576063.105036:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576063.105040:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576063.105044:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576063.108332:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576063.108339:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576063.108342:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576063.108347:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576063.108351:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576063.108354:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576063.108366:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576063.108369:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576063.108372:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576063.809097:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576063.809100:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576063.809103:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576064.128946:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576064.128952:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576064.128955:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576064.128960:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576064.128965:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576064.128969:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576064.128972:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576064.128976:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576064.128979:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576064.129164:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576064.129182:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576064.129197:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576064.129201:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576064.129203:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576064.129205:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576064.129207:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576064.129209:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576064.129211:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576064.833100:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576064.833104:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576064.833108:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788576065.408920:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311425261, at 4311425280 00000100:00000010:0.0:1788576065.408939:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674960. 00000020:00000001:0.0:1788576065.857171:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576065.857175:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576065.857178:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576066.180796:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576066.180800:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576066.180802:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576066.180804:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576066.180807:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576066.180826:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576066.180828:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576066.180830:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576066.180832:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576066.688879:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576066.688890:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788576066.688893:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576066.688896:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576066.688901:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576066.688921:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16754 00000400:00000200:0.0:1788576066.688931:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576066.688937:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576066.688940:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576066.688943:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.688962:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576066.688965:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.688968:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576066.688971:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576066.688973:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788576066.688975:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576066.688978:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788576066.688981:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788576066.688983:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788576066.688985:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.688987:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576066.688989:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.688991:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576066.688993:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788576066.688996:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576066.688998:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576066.689001:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576066.689004:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576066.689006:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.689008:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788576066.689011:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576066.689027:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3800. 00000020:00000040:0.0:1788576066.689030:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788576066.689034:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576066.689035:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.689038:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576066.689041:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576066.689045:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576066.689051:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788576066.689057:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576066.689065:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3800 x1875440516900992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576066.689084:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576066.689087:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576066.689092:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae7d5dea] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3800 x1875440516900992/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576066.689115:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576066.689118:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576066.689121:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788576066.689123:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576066.689124:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576066.689127:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576066.689129:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16754 00000400:00000200:0.0:1788576066.689133:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576066.689136:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576066.689138:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576066.689141:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689143:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576066.689145:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689147:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576066.689150:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576066.689151:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788576066.689153:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576066.689156:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788576066.689158:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788576066.689160:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788576066.689163:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689166:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576066.689168:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689170:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576066.689179:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788576066.689185:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689188:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576066.689205:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576066.689211:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576066.689216:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.689220:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576066.689226:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576066.689232:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.689235:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576066.689242:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3480. 00000020:00000040:0.0:1788576066.689248:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576066.689253:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576066.689257:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.689260:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576066.689262:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576066.689268:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576066.689276:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788576066.689281:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576066.689288:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3480 x1875440516901120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576066.689312:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576066.689316:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576066.689321:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000080ad2b3a] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3480 x1875440516901120/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576066.689338:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576066.689343:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576066.689351:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788576066.689355:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576066.689357:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576066.689364:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576066.689377:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16754 00000400:00000200:0.0:1788576066.689386:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576066.689391:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576066.689395:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576066.689401:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689405:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576066.689411:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689415:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576066.689419:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576066.689421:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788576066.689425:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576066.689428:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788576066.689434:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788576066.689438:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788576066.689444:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689450:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576066.689454:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689457:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576066.689462:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788576066.689469:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689475:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576066.689479:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576066.689483:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576066.689488:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.689493:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576066.689495:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576066.689500:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.689505:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576066.689509:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1880. 00000020:00000040:0.0:1788576066.689514:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576066.689518:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576066.689525:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.689531:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576066.689535:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576066.689538:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576066.689544:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788576066.689550:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576066.689559:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1880 x1875440516901248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576066.689571:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576066.689574:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576066.689582:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005137afc1] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1880 x1875440516901248/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576066.689601:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576066.689604:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576066.689613:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788576066.689617:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576066.689619:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576066.689623:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576066.689629:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16754 00000400:00000200:0.0:1788576066.689640:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576066.689646:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576066.689650:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576066.689653:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689658:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576066.689663:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689667:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576066.689671:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576066.689672:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788576066.689676:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576066.689679:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788576066.689681:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788576066.689691:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788576066.689699:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689704:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576066.689710:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689715:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576066.689720:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788576066.689728:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689734:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576066.689738:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576066.689742:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576066.689747:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.689752:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576066.689757:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576066.689774:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.689779:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576066.689787:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0000. 00000020:00000040:0.0:1788576066.689791:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576066.689796:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576066.689799:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.689802:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576066.689806:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576066.689842:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576066.689850:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788576066.689857:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576066.689866:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0000 x1875440516901376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576066.689882:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576066.689887:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576066.689895:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5acb0d9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0000 x1875440516901376/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576066.689912:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576066.689922:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576066.689930:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788576066.689933:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576066.689936:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576066.689941:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576066.689947:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16754 00000400:00000200:0.0:1788576066.689954:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576066.689962:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576066.689966:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576066.689972:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689977:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576066.689981:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.689985:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576066.689990:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576066.689993:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788576066.689996:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576066.689998:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788576066.690000:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788576066.690005:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788576066.690011:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.690014:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576066.690016:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576066.690017:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576066.690021:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788576066.690026:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576066.690030:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576066.690033:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576066.690038:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576066.690042:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.690046:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576066.690054:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576066.690058:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.690062:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576066.690068:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2a00. 00000020:00000040:0.0:1788576066.690073:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576066.690079:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576066.690082:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576066.690085:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576066.690087:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576066.690089:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576066.690094:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788576066.690100:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576066.690109:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2a00 x1875440516901504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576066.690122:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576066.690127:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576066.690135:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001a2f8cdc] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2a00 x1875440516901504/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576066.690151:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576066.690159:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16764) 00000100:00000001:0.0:1788576067.200954:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576067.200962:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576067.200964:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576067.200969:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576067.200973:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576067.200976:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576067.200978:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576067.200981:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576067.200983:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576067.201004:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576067.201007:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576067.201008:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576067.201010:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576067.201013:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576067.201028:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576067.201030:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576067.201031:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576067.201034:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576067.905048:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576067.905051:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576067.905053:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576068.225010:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576068.225015:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576068.225019:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576068.225024:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576068.225028:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576068.225032:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576068.225034:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576068.225036:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576068.225038:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576068.929025:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576068.929029:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576068.929032:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576069.250502:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576069.250507:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576069.250509:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576069.250513:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576069.250516:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576069.250518:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576069.250521:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576069.250523:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576069.250525:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576069.953042:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576069.953045:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576069.953048:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576070.272912:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576070.272916:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576070.272919:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576070.272926:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576070.272932:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576070.272937:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576070.272950:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576070.272972:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576070.272976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576070.273065:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576070.273068:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576070.273069:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576070.273071:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576070.273072:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576070.273074:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576070.273388:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576070.273391:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576070.273393:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576070.528971:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311430381, at 4311430400 00000100:00000010:0.0:1788576070.528996:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674300. 00000020:00000001:0.0:1788576070.977143:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576070.977146:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576070.977149:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576071.296963:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576071.296972:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576071.296974:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576071.296979:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576071.296983:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576071.296987:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576071.296990:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576071.296993:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576071.296996:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576071.299420:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576071.299424:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576071.299425:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576071.299428:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576071.299430:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576071.299432:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576071.299434:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576071.299436:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576071.299437:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576072.321051:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576072.321057:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576072.321061:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576072.321075:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576072.321082:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576072.321087:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576072.321091:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576072.321093:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576072.321096:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576073.025237:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576073.025241:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576073.025244:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576073.346298:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576073.346302:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576073.346304:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576073.346307:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576073.346309:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576073.346311:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576073.346314:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576073.346316:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576073.346319:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576074.049062:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576074.049065:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576074.049068:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576074.368984:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576074.368993:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576074.368997:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576074.369005:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576074.369011:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576074.369017:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576074.369023:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576074.369026:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576074.369029:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576074.369094:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576074.369097:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576074.369099:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576074.369101:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576074.369103:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576074.369105:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576074.369107:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576074.369108:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576074.369132:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576074.369149:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576074.369151:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576074.369153:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576074.369155:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576074.369157:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576074.369159:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576074.369161:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576074.369162:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576074.369164:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576075.072986:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576075.072989:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576075.072993:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576075.392954:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576075.392964:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576075.392968:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576075.392973:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576075.392976:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576075.392979:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576075.392981:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576075.392983:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576075.392986:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576075.393003:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576075.393008:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576075.393011:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576075.393015:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576075.393019:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576075.393022:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576075.393033:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576075.393036:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576075.393038:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576075.648935:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311435502, at 4311435520 00000100:00000010:0.0:1788576075.648950:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674180. 00000020:00000001:0.0:1788576076.097109:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576076.097112:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576076.097115:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576076.416935:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576076.416962:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576076.416965:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576076.416969:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576076.416975:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576076.416980:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576076.416988:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576076.416992:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576076.416995:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576076.417094:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576076.417097:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576076.417099:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576076.417101:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576076.417104:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576076.417106:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576076.417111:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576076.417115:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576076.417120:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576077.440929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576077.440934:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576077.440938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576077.440944:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576077.440958:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576077.440964:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576077.440970:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576077.440973:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576077.440976:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576078.145043:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576078.145047:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576078.145051:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576078.465242:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576078.465246:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576078.465247:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576078.465250:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576078.465252:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576078.465254:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576078.465255:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576078.465257:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576078.465259:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576078.466172:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576078.466175:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576078.466177:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576078.466180:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576078.466182:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576078.466184:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576078.466186:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576078.466187:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576078.466189:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576079.169115:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576079.169118:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576079.169121:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576079.488957:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576079.488964:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576079.488967:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576079.488972:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576079.488976:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576079.488980:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576079.488983:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576079.488987:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576079.488994:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576079.490289:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576079.490292:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576079.490294:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576079.490299:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576079.490301:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576079.490303:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576079.490305:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576079.490307:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576079.490309:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576080.193211:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576080.193213:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576080.193215:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576080.513238:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576080.513242:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576080.513244:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576080.513247:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576080.513250:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576080.513271:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576080.513274:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576080.513277:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576080.513281:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576080.517023:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576080.517029:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576080.517031:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576080.517035:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576080.517038:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576080.517041:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576080.517043:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576080.517046:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576080.517048:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576080.768931:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311440621, at 4311440640 00000100:00000010:0.0:1788576080.768950:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916742a0. 00000020:00000001:0.0:1788576081.217274:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576081.217278:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576081.217281:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576081.539326:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576081.539329:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576081.539331:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576081.539335:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576081.539338:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576081.539341:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576081.539343:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576081.539345:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576081.539347:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576082.561072:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576082.561075:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576082.561077:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576082.561079:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576082.561081:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576082.561083:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576082.561096:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576082.561097:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576082.561100:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576082.562708:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576082.562723:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576082.562723:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576082.562725:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576082.562726:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576082.562727:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576082.562728:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576082.562730:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576082.562731:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576083.265078:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576083.265081:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576083.265084:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576083.584886:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576083.584890:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576083.584892:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576083.584896:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576083.584905:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576083.584907:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576083.585099:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576083.585102:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576083.585104:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576084.289124:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576084.289128:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576084.289131:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576084.609065:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576084.609069:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576084.609070:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576084.609073:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576084.609076:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576084.609078:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576084.609079:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576084.609081:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576084.609084:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576085.313270:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576085.313273:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576085.313276:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576085.632919:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576085.632927:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576085.632930:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576085.632948:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576085.632950:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576085.632952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576085.632959:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576085.632960:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576085.632962:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576085.633025:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576085.633030:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576085.633034:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576085.633040:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576085.633046:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576085.633051:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576085.633057:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576085.633060:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576085.633062:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576085.888945:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311445745, at 4311445760 00000100:00000010:0.0:1788576085.888961:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916746c0. 00000020:00000001:0.0:1788576086.337058:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576086.337062:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576086.337064:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576086.656925:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576086.656933:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576086.656935:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576086.656940:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576086.656944:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576086.656948:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576086.656959:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576086.656961:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576086.656963:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576087.682823:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576087.682826:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576087.682827:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576087.682830:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576087.682832:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576087.682834:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576087.682835:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576087.682837:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576087.682848:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576088.385139:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576088.385143:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576088.385146:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576088.704960:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576088.704964:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576088.704966:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576088.704970:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576088.705155:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576088.705158:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576088.705159:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576088.705161:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576088.705163:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576088.705351:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576088.705354:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576088.705355:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576088.705357:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576088.705360:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576088.705361:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576088.705363:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576088.705364:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576088.705366:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576089.411637:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576089.411640:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576089.411643:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576089.732561:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576089.732566:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576089.732568:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576089.732571:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576089.732574:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576089.732577:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576089.732580:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576089.732581:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576089.732583:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576090.433172:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576090.433176:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576090.433179:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788576091.008923:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311450862, at 4311450880 00000100:00000010:0.0:1788576091.008950:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4916743c0. 00000020:00000001:0.0:1788576091.457046:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576091.457050:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576091.457061:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576091.777217:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576091.777220:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576091.777221:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576091.777224:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576091.777242:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576091.777244:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576091.777246:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576091.777248:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576091.777250:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576092.802166:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576092.802171:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576092.802173:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576092.802178:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576092.802184:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576092.802190:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576092.802195:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576092.802198:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576092.802200:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576093.505108:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576093.505111:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576093.505114:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576093.832407:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576093.832413:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576093.832415:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576093.832420:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576093.832423:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576093.832426:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576093.832429:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576093.832431:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576093.832434:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576094.531329:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576094.531334:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576094.531338:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788576095.554447:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576095.554451:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576095.554455:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576095.875200:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576095.875203:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576095.875205:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576095.875208:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576095.875209:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576095.875212:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576095.875213:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576095.875215:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576095.875217:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576096.129100:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311455981, at 4311456000 00000100:00000010:0.0:1788576096.129107:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf180. 00000020:00000001:0.0:1788576096.577071:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576096.577074:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576096.577086:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576096.897036:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576096.897368:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576096.897369:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576096.897372:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576096.897375:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576096.897378:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576096.897571:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576096.897574:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576096.897577:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576097.920941:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576097.920953:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576097.920958:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576097.920969:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576097.920979:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576097.920987:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576097.921004:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576097.921008:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576097.921012:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576097.921300:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576097.921304:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576097.921328:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576097.921331:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576097.921333:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576097.921335:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576097.921337:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576097.921339:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576097.921341:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576098.625142:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576098.625146:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576098.625150:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576098.945171:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576098.945178:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576098.945180:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576098.945186:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576098.945189:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576098.945192:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576098.945195:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576098.945197:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576098.945200:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576099.649332:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576099.649338:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576099.649342:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576099.969983:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576099.969989:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576099.969990:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576099.969994:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576099.969996:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576099.969999:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576099.970001:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576099.970003:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576099.970005:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576099.970492:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576099.970495:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576099.970497:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576099.970499:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576099.970501:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576099.970504:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576099.970506:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576099.970520:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576099.970522:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576100.673333:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576100.673338:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576100.673342:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576100.993233:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576100.993236:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576100.993238:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576100.993241:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576100.993243:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576100.993245:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576100.993247:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576100.993249:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576100.993251:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576100.993261:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576100.993264:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576100.993265:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576100.993267:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576100.993269:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576100.993270:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576100.993272:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576100.993274:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576100.993276:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576101.251480:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311461102, at 4311461122 00000100:00000010:0.0:1788576101.251495:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf2a0. 00000020:00000001:0.0:1788576101.697265:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576101.697270:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576101.697273:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576102.017761:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576102.017765:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576102.017767:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576102.017770:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576102.017773:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576102.017776:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576102.017778:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576102.017780:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576102.017782:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576103.745388:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576103.745393:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576103.745397:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576104.065489:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576104.065495:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576104.065497:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576104.065503:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576104.065509:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576104.065514:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576104.065521:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576104.065526:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576104.065530:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576104.770231:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576104.770238:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576104.770243:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576105.088998:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576105.089009:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576105.089014:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576105.089022:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576105.089027:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576105.089033:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576105.089038:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576105.089044:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576105.089051:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576105.793141:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576105.793144:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576105.793147:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576106.112965:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576106.112973:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576106.112975:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576106.112981:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576106.112985:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576106.112989:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576106.112991:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576106.112994:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576106.112998:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576106.368936:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311466221, at 4311466240 00000100:00000010:0.0:1788576106.368954:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf780. 00000020:00000001:0.0:1788576106.817342:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576106.817345:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576106.817348:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1788576107.649314:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788576107.649324:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788576107.649340:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1788576107.649470:0:2392:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1788576107.649475:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576107.649487:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1788576107.649491:0:2392:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1788576107.649987:0:2392:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576108.162045:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576108.162048:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576108.162049:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576108.162052:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576108.162054:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576108.162057:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576108.162058:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576108.162060:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576108.162062:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576108.867454:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576108.867459:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576108.867463:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788576109.889069:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576109.889072:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576109.889075:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576110.209332:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576110.209335:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576110.209336:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576110.209339:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576110.209341:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576110.209342:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576110.209344:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576110.209345:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576110.209347:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576110.913206:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576110.913209:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576110.913219:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576111.232944:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576111.232951:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576111.232955:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576111.232963:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576111.232970:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576111.232976:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576111.232982:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576111.232985:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576111.232988:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576111.488901:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311471342, at 4311471360 00000100:00000010:0.0:1788576111.488917:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cfae0. 00000020:00000001:0.0:1788576111.937016:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576111.937019:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576111.937022:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576112.257075:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576112.257078:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576112.257080:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576112.257082:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576112.257084:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576112.257086:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576112.257088:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576112.257089:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576112.257091:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576112.258923:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576112.258925:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576112.258927:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576112.258929:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576112.258931:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576112.258933:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576112.258934:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576112.258936:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576112.258937:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576113.281524:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576113.281528:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576113.281530:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576113.281532:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576113.281550:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576113.281552:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576113.281554:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576113.281555:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576113.281557:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576113.985366:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576113.985369:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576113.985372:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576114.305565:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576114.305569:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576114.305570:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576114.305573:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576114.305576:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576114.305578:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576114.305580:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576114.305581:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576114.305583:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576115.010544:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576115.010548:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576115.010552:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576115.328948:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576115.328957:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576115.328960:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576115.328969:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576115.328976:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576115.328983:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576115.328996:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576115.328999:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576115.329003:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576116.033642:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576116.033645:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576116.033649:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576116.352904:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576116.352911:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576116.352914:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576116.352920:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576116.352924:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576116.352927:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576116.352944:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576116.352947:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576116.352949:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576116.353463:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576116.353467:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576116.353469:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576116.353473:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576116.353475:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576116.353478:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576116.353583:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576116.353585:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576116.353588:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576116.608926:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311476464, at 4311476480 00000100:00000010:0.0:1788576116.608942:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9cf5a0. 00000020:00000001:0.0:1788576117.060013:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576117.060016:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576117.060019:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576117.377793:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576117.377797:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576117.377799:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576117.377803:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576117.377829:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576117.377835:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576117.377842:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576117.377846:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576117.377848:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576117.888963:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576117.889239:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788576117.889242:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576117.889245:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576117.889249:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576117.889256:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16805 00000400:00000200:0.0:1788576117.889265:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576117.889271:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576117.889274:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576117.889295:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.889298:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576117.889301:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.889303:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576117.889307:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576117.889309:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788576117.889311:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576117.889314:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788576117.889316:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788576117.889319:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788576117.889384:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.889386:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576117.889388:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.889390:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576117.889393:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788576117.889397:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576117.889400:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576117.889403:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576117.889406:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576117.889409:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.889411:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788576117.889414:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576117.889421:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c3b80. 00000020:00000040:0.0:1788576117.889425:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788576117.889428:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576117.889430:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.889433:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576117.889436:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576117.889440:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576117.889447:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788576117.889454:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576117.889462:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c3b80 x1875440516907392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576117.889477:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576117.889481:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576117.889486:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e281ea32] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c3b80 x1875440516907392/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576117.889514:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576117.889518:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576117.889522:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788576117.889523:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576117.889525:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576117.889527:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576117.889530:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16805 00000400:00000200:0.0:1788576117.889535:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576117.889539:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576117.889542:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576117.889545:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.889548:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576117.889555:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.889559:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576117.889565:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576117.889570:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788576117.889573:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576117.889577:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788576117.889582:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788576117.889592:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788576117.889604:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.889607:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576117.889609:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.889611:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576117.889632:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788576117.889642:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576117.889652:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576117.889660:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576117.889668:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576117.889673:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.889678:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576117.889688:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576117.889700:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.889709:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576117.889718:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1180. 00000020:00000040:0.0:1788576117.889724:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576117.889729:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576117.889731:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.889737:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576117.889745:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576117.889749:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576117.889764:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788576117.889774:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576117.889787:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1180 x1875440516907520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576117.889800:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576117.889805:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576117.891059:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e919691] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1180 x1875440516907520/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576117.891084:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576117.891088:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576117.891091:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788576117.891093:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576117.891095:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576117.891098:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576117.891111:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16805 00000400:00000200:0.0:1788576117.891118:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576117.891123:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576117.891127:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576117.891136:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.891141:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576117.891149:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.891153:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576117.891165:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576117.891170:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788576117.891178:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576117.891183:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788576117.891191:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788576117.891199:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788576117.891213:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.891220:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576117.891231:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.891240:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576117.891248:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788576117.891257:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576117.891267:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576117.891272:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576117.891280:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576117.891290:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.891297:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576117.891302:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576117.891309:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.891313:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576117.891318:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1500. 00000020:00000040:0.0:1788576117.891331:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576117.891341:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576117.891351:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.891359:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576117.891363:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576117.891369:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576117.891381:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788576117.891390:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576117.891399:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1500 x1875440516907648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576117.891434:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576117.891438:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576117.891450:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000333c7d2d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1500 x1875440516907648/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576117.891488:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576117.891494:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576117.891503:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788576117.891510:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576117.891520:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576117.891534:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576117.891543:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16805 00000400:00000200:0.0:1788576117.891552:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576117.891563:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576117.891571:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576117.891583:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.891591:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576117.891602:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.891609:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576117.891617:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576117.891620:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788576117.891624:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576117.891629:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788576117.891637:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788576117.891653:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788576117.891669:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.891679:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576117.891688:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.891699:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576117.891710:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788576117.891719:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576117.891733:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576117.891741:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576117.891747:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576117.891759:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.891766:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576117.891773:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576117.891783:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.891793:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576117.891803:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0a80. 00000020:00000040:0.0:1788576117.891834:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576117.891842:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576117.891845:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.891853:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576117.891859:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576117.891862:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576117.891870:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788576117.891884:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576117.891906:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0a80 x1875440516907776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576117.891924:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576117.891930:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576117.891940:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007add588] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0a80 x1875440516907776/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576117.891968:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576117.891981:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576117.891988:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788576117.891992:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576117.891997:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576117.892004:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576117.892011:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16805 00000400:00000200:0.0:1788576117.892022:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576117.892031:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576117.892037:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576117.892042:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.892047:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576117.892053:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.892059:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576117.892067:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576117.892069:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788576117.892070:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576117.892072:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788576117.892074:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788576117.892079:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788576117.892087:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.892096:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576117.892102:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576117.892104:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576117.892107:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788576117.892113:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576117.892121:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576117.892128:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576117.892132:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576117.892139:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.892148:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576117.892156:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576117.892160:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.892165:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576117.892171:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0380. 00000020:00000040:0.0:1788576117.892175:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576117.892177:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576117.892178:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576117.892179:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576117.892180:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576117.892185:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576117.892190:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788576117.892194:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576117.892206:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0380 x1875440516907904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576117.892220:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576117.892225:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576117.892234:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001b9d0da2] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0380 x1875440516907904/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576117.892254:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576117.892260:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16815) 00000100:00000001:0.0:1788576118.400960:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576118.400969:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576118.400973:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576118.400980:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576118.400986:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576118.400992:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576118.400995:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576118.401006:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576118.401011:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576118.402576:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576118.402581:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576118.402582:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576118.402586:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576118.402589:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576118.402606:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576118.402608:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576118.402609:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576118.402612:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576119.105167:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576119.105170:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576119.105173:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576119.424918:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576119.424925:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576119.424927:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576119.424934:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576119.424940:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576119.424946:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576119.424952:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576119.424956:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576119.424962:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576119.424977:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576119.424979:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576119.424981:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576119.424985:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576119.424990:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576119.424994:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576119.424999:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576119.425001:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576119.425003:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576120.129763:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576120.129768:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576120.129772:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788576121.154639:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576121.154643:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576121.154646:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576121.473021:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576121.473026:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576121.473031:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576121.473040:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576121.473049:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576121.473054:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576121.473061:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576121.473084:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576121.473086:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576121.728970:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311481582, at 4311481600 00000100:00000010:0.0:1788576121.728987:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abae0. 00000020:00000001:0.0:1788576122.178065:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576122.178072:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576122.178076:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576122.499498:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576122.499506:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576122.499510:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576122.499516:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576122.499520:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576122.499523:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576122.499526:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576122.499528:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576122.499531:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576123.523800:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576123.523805:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576123.523806:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576123.523826:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576123.523828:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576123.523830:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576123.523832:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576123.523834:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576123.523836:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576124.225394:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576124.225402:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576124.225408:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576124.545106:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576124.545112:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576124.545114:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576124.545118:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576124.545122:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576124.545127:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576124.545133:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576124.545137:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576124.545144:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576125.249208:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576125.249212:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576125.249215:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1788576126.273018:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576126.273023:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576126.273026:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788576126.848925:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311486702, at 4311486720 00000100:00000010:0.0:1788576126.848950:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab3c0. 00000020:00000001:0.0:1788576127.297003:0:241699:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576127.297006:0:241699:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576127.297009:0:241699:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576128.642562:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576128.642567:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576128.642569:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576128.642573:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576128.642579:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576128.642583:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576128.642592:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576128.642594:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576128.642597:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576129.347471:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576129.347477:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576129.347481:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576129.664959:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576129.664968:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576129.664970:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576129.664976:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576129.664981:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576129.664984:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576129.664987:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576129.664990:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576129.664993:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576129.665399:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576129.665402:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576129.665403:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576129.665405:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576129.665407:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576129.665422:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576129.665424:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576129.665425:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576129.665427:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576130.369211:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576130.369215:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576130.369219:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576130.688934:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576130.688944:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576130.688948:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576130.688955:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576130.688960:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576130.688965:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576130.688969:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576130.688973:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576130.688975:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576130.689023:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576130.689026:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576130.689027:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576130.689029:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576130.689031:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576130.689033:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576130.689468:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576130.689471:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576130.689474:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576131.393137:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576131.393139:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576131.393142:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576131.714448:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576131.714453:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576131.714455:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576131.714459:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576131.714462:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576131.714465:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576131.714467:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576131.714469:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576131.714471:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576131.968934:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311491821, at 4311491840 00000100:00000010:0.0:1788576131.968977:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab840. 00000100:00000001:0.0:1788576132.737046:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576132.737136:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576132.737139:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576132.737143:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576132.737145:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576132.737147:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576132.737149:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576132.737151:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576132.737152:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576132.742752:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576132.742756:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576132.742758:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576132.742761:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576132.742762:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576132.742764:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576132.742766:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576132.742767:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576132.742769:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576133.761010:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576133.761015:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576133.761016:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576133.761019:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576133.761022:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576133.761025:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576133.761029:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576133.761032:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576133.761036:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576134.465270:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576134.465274:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576134.465278:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576134.785012:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576134.785022:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576134.785027:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576134.785036:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576134.785043:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576134.785048:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576134.785067:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576134.785070:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576134.785074:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576134.791660:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576134.791667:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576134.791671:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576134.791677:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576134.791682:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576134.791685:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576134.791687:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576134.791689:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576134.791691:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576135.808963:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576135.808969:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576135.808972:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576135.808977:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576135.808982:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576135.808986:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576135.808991:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576135.808994:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576135.808998:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576135.809382:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576135.809385:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576135.809386:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576135.809389:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576135.809391:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576135.809393:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576135.809796:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576135.809800:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576135.809805:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576136.513719:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576136.513722:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576136.513725:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576136.833206:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576136.833209:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576136.833210:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576136.833214:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576136.833216:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576136.833232:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576136.833234:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576136.833235:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576136.833237:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576136.839742:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576136.839749:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576136.839751:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576136.839756:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576136.839759:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576136.839762:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576136.839764:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576136.839766:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576136.839769:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576137.091448:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311496944, at 4311496962 00000100:00000010:0.0:1788576137.091460:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abcc0. 00000100:00000001:0.0:1788576137.857479:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576137.857483:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576137.857485:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576137.857489:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576137.857492:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576137.857494:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576137.857495:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576137.857497:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576137.857498:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576138.882646:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576138.882649:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576138.882651:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576138.882653:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576138.882656:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576138.882657:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576138.882659:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576138.882661:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576138.882663:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576139.585173:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576139.585176:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576139.585179:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576139.905017:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576139.905026:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576139.905027:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576139.905030:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576139.905033:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576139.905035:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576139.908300:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576139.908304:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576139.908306:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576140.928926:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576140.928935:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576140.928938:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576140.928947:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576140.928953:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576140.928958:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576140.928964:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576140.928967:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576140.928969:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576141.633412:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576141.633416:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576141.633419:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576141.952927:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576141.952938:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576141.952940:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576141.955932:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576141.955937:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576141.955941:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576141.955942:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576141.955944:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576141.955946:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576142.209025:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311502062, at 4311502080 00000100:00000010:0.0:1788576142.209042:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674180. 00000100:00000001:0.0:1788576142.979379:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576142.979382:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576142.979383:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576142.979396:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576142.979399:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576142.979402:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576142.979410:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576142.979412:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576142.979415:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576144.705459:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576144.705463:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576144.705466:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576145.025037:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576145.025041:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576145.025044:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576145.025049:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576145.025053:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576145.025057:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576145.027200:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576145.027203:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576145.027206:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788576145.537078:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788576145.537087:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788576145.537091:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:0.0:1788576146.049190:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576146.049192:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576146.049194:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576146.049197:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576146.049199:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576146.049202:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576146.052820:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576146.052824:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576146.052826:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576146.753510:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576146.753514:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576146.753517:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576147.073203:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576147.073206:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576147.073207:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576147.073209:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576147.073211:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576147.073213:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576147.073215:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576147.073222:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576147.073225:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576147.329240:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311507182, at 4311507200 00000100:00000010:0.0:1788576147.329247:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674300. 00000100:00000001:0.0:1788576148.097336:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576148.097342:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576148.097344:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576148.097349:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576148.097353:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576148.097355:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576148.097358:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576148.097360:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576148.097363:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576148.098356:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576148.098361:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576148.098362:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576148.098366:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576148.098369:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576148.098371:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576148.098374:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576148.098376:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576148.098378:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576149.120898:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576149.120901:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576149.120903:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576149.120905:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576149.120908:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576149.120913:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576149.121016:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576149.121018:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576149.121020:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576149.121047:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576149.121051:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576149.121052:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576149.121054:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576149.121056:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576149.121058:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576149.121073:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576149.121074:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576149.121077:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576149.827162:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576149.827167:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576149.827173:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576150.145046:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576150.145055:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576150.145057:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576150.145063:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576150.145068:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576150.145071:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576150.145075:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576150.145078:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576150.145081:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576151.170128:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576151.170132:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576151.170133:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576151.170136:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576151.170139:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576151.170141:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576151.170143:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576151.170145:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576151.170147:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576151.873147:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576151.873150:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576151.873154:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788576152.448973:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311512303, at 4311512320 00000100:00000010:0.0:1788576152.449003:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abba0. 00000100:00000001:0.0:1788576153.217003:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576153.217010:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576153.217014:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576153.217019:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576153.217022:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576153.217025:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576153.217028:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576153.217030:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576153.217053:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576154.241267:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576154.241271:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576154.241273:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576154.241280:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576154.241285:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576154.241290:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576154.241293:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576154.241296:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576154.241300:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576154.947072:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576154.947079:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576154.947085:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576155.265075:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576155.265087:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576155.265090:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576155.265097:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576155.265102:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576155.265107:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576155.265111:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576155.265114:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576155.265117:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576155.267017:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576155.267029:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576155.267031:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576155.267039:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576155.267044:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576155.267048:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576155.267052:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576155.267055:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576155.267058:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576156.289201:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576156.289204:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576156.289206:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576156.289209:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576156.289211:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576156.289213:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576156.295644:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576156.295667:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576156.295670:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576156.993124:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576156.993129:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576156.993133:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576157.313946:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576157.313951:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576157.313952:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576157.313956:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576157.313959:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576157.313961:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576157.313963:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576157.313966:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576157.313968:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576157.570161:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311517422, at 4311517441 00000100:00000010:0.0:1788576157.570168:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abde0. 00000100:00000001:0.0:1788576158.337072:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576158.337075:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576158.337077:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576158.337082:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576158.337086:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576158.337090:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576158.337095:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576158.337098:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576158.337101:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576158.337117:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576158.337120:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576158.337121:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576158.337123:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576158.337125:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576158.337127:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576158.346182:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576158.346193:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576158.346201:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576159.361268:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576159.361272:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576159.361273:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576159.361297:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576159.361299:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576159.361301:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576159.362885:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576159.362889:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576159.362891:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576160.065427:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576160.065457:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576160.065461:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576160.386204:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576160.386209:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576160.386211:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576160.386214:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576160.386217:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576160.386220:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576160.386222:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576160.386224:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576160.386227:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576160.386789:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576160.386792:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576160.386793:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576160.386796:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576160.386798:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576160.386800:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576160.386801:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576160.386802:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576160.386804:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576161.409004:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576161.409018:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576161.409020:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576161.409028:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576161.409033:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576161.409038:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576161.409042:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576161.409045:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576161.409048:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576161.410834:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576161.410837:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576161.410862:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576161.410865:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576161.410867:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576161.410869:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576161.410871:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576161.410873:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576161.410874:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576162.116551:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576162.116559:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576162.116565:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576162.433023:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576162.433034:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576162.433037:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576162.433044:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576162.433050:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576162.433054:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576162.433058:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576162.433061:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576162.433064:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576162.433324:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576162.433326:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576162.433327:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576162.433329:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576162.433330:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576162.433332:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576162.433333:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576162.433334:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576162.433336:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576162.433485:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576162.433488:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576162.433489:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576162.433491:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576162.433493:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576162.433495:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576162.433498:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576162.433499:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576162.433501:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576162.689090:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311522543, at 4311522560 00000100:00000010:0.0:1788576162.689121:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab600. 00000100:00000001:0.0:1788576163.457416:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576163.457419:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576163.457421:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576163.457424:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576163.457426:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576163.457427:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576163.457429:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576163.457431:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576163.457433:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576163.458829:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576163.458832:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576163.458833:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576163.458835:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576163.458837:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576163.458839:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576163.458841:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576163.458842:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576163.458843:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576164.480969:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576164.480977:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576164.480979:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576164.480984:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576164.480988:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576164.480992:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576164.480994:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576164.480997:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576164.480999:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576164.481298:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576164.481300:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576164.481302:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576164.481304:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576164.481306:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576164.481308:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576164.481309:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576164.481310:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576164.481312:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576165.185631:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576165.185634:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576165.185637:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576165.504927:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576165.504939:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576165.504942:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576165.504947:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576165.504951:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576165.504955:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576165.504958:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576165.504961:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576165.504963:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576167.233005:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576167.233008:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576167.233011:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576167.553704:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576167.553707:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576167.553709:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576167.553712:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576167.553714:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576167.553716:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576167.553718:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576167.553719:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576167.553721:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576167.808911:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311527662, at 4311527680 00000100:00000010:0.0:1788576167.808930:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9abb40. 00000100:00000001:0.0:1788576168.577003:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576168.577008:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576168.577011:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576168.577016:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576168.577019:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576168.577021:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576168.577022:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576168.577023:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576168.577025:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576168.577180:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576168.577183:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576168.577210:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576168.577213:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576168.577217:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576168.577220:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576168.577222:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576168.577225:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576168.577226:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576169.601152:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576169.601161:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576169.601164:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576169.601171:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576169.601176:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576169.601180:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576169.601190:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576169.601193:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576169.601196:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576169.603645:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576169.603649:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576169.603650:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576169.603652:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576169.603654:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576169.603655:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576169.608383:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576169.608391:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576169.608398:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576170.305354:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576170.305358:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576170.305362:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576170.624963:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576170.624974:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576170.624979:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576170.624989:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576170.624996:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576170.625002:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576170.625072:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576170.625075:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576170.625077:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576170.627419:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576170.627459:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576170.627461:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576170.627467:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576170.627471:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576170.627475:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576170.627478:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576170.627480:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576170.627482:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576171.649267:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576171.649271:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576171.649273:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576171.649276:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576171.649278:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576171.649280:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576171.649283:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576171.649284:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576171.649287:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576172.353305:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576172.353318:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576172.353323:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576172.672995:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576172.673007:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576172.673009:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576172.673016:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576172.673021:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576172.673025:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576172.673029:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576172.673032:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576172.673036:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576172.674225:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576172.674230:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576172.674233:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576172.674237:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576172.674241:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576172.674244:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576172.674246:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576172.674248:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576172.674251:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576172.675382:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576172.675388:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576172.675390:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576172.675395:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576172.675399:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576172.675403:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576172.675405:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576172.675407:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576172.675419:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576172.931122:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311532783, at 4311532802 00000100:00000010:0.0:1788576172.931142:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab720. 00000100:00000001:0.0:1788576174.721189:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576174.721194:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576174.721196:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576174.721199:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576174.721203:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576174.721208:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576174.721212:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576174.721216:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576174.721219:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576175.426120:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576175.426127:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576175.426132:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576175.747797:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576175.747806:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576175.747882:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576175.747888:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576175.747892:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576175.747897:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576175.747900:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576175.747903:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576175.747906:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576176.769191:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576176.769195:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576176.769196:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576176.769199:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576176.769201:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576176.769221:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576176.769223:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576176.769225:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576176.769227:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576176.769555:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576176.769557:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576176.769558:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576176.769560:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576176.769562:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576176.769563:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576176.769565:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576176.769566:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576176.769568:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576177.478517:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576177.478524:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576177.478529:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576177.793745:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576177.793753:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576177.793755:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576177.793761:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576177.793766:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576177.793769:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576177.793772:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576177.793774:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576177.793777:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576177.793970:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576177.793972:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576177.793974:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576177.793976:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576177.793978:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576177.793980:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576177.793982:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576177.793983:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576177.793986:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576178.049319:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311537907, at 4311537920 00000100:00000010:0.0:1788576178.049326:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab2a0. 00000100:00080000:0.0:1788576179.333874:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576179.333918:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788576179.333921:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576179.333927:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576179.333933:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576179.333944:0:231961:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16866 00000400:00000200:0.0:1788576179.333956:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576179.333964:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576179.333968:0:231961:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576179.333971:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.333975:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576179.333978:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.333981:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576179.333986:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576179.333989:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788576179.333992:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576179.333996:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788576179.333999:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788576179.334002:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788576179.334005:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.334007:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576179.334010:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.334012:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576179.334015:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788576179.334019:0:231961:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576179.334021:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576179.334025:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576179.334029:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576179.334033:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.334035:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788576179.334038:0:231961:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576179.334055:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788576179.334067:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788576179.334072:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576179.334074:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.334077:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576179.334081:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576179.334087:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576179.334096:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9d800. 00000100:00000001:0.0:1788576179.334104:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576179.334115:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516915072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576179.334126:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576179.334132:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576179.334138:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516915072/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576179.334679:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576179.334683:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576179.334687:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788576179.334689:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576179.334690:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576179.334693:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576179.334696:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16866 00000400:00000200:0.0:1788576179.334701:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576179.334703:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576179.334706:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576179.334709:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.334711:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576179.334713:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.334715:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576179.334718:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576179.334719:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788576179.334721:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576179.334730:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788576179.334732:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788576179.334734:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788576179.334738:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.334740:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576179.334742:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.334744:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576179.334746:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788576179.334749:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576179.334752:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576179.334754:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576179.334761:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576179.334763:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.334766:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576179.334771:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576179.334775:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.334777:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576179.334781:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788576179.334784:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576179.334786:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576179.334788:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.334790:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576179.334792:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576179.334794:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576179.334800:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9dc00. 00000100:00000001:0.0:1788576179.334804:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576179.334851:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516915200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576179.334859:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576179.334861:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576179.334865:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516915200/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576179.343515:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576179.343532:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576179.343538:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788576179.343541:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576179.343545:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576179.343551:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576179.343559:0:231961:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16866 00000400:00000200:0.0:1788576179.343571:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576179.343578:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576179.343582:0:231961:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576179.343585:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343589:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576179.343591:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343594:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576179.343598:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576179.343601:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788576179.343604:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576179.343608:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788576179.343611:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788576179.343614:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788576179.343617:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343619:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576179.343621:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343623:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576179.343627:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788576179.343630:0:231961:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343633:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576179.343658:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576179.343671:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576179.343675:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.343678:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576179.343683:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576179.343686:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.343688:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576179.343696:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788576179.343702:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576179.343706:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576179.343709:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.343712:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576179.343716:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576179.343721:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576179.343731:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9fc00. 00000100:00000001:0.0:1788576179.343740:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576179.343753:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516915328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576179.343764:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576179.343770:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576179.343775:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516915328/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576179.343787:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576179.343790:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576179.343794:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788576179.343795:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576179.343797:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576179.343800:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576179.343803:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16866 00000400:00000200:0.0:1788576179.343807:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576179.343827:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576179.343830:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576179.343838:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343840:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576179.343843:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343845:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576179.343847:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576179.343848:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788576179.343850:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576179.343852:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788576179.343854:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788576179.343856:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788576179.343859:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343861:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576179.343862:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343865:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576179.343867:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788576179.343869:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343872:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576179.343874:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576179.343878:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576179.343880:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.343882:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576179.343885:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576179.343887:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.343889:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576179.343892:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788576179.343895:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576179.343897:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576179.343898:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.343900:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576179.343902:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576179.343904:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576179.343908:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f400. 00000100:00000001:0.0:1788576179.343916:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576179.343923:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516915456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576179.343930:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576179.343932:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576179.343936:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516915456/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576179.343943:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576179.343946:0:231961:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576179.343950:0:231961:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788576179.343951:0:231961:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576179.343953:0:231961:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576179.343956:0:231961:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576179.343958:0:231961:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16866 00000400:00000200:0.0:1788576179.343962:0:231961:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576179.343965:0:231961:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576179.343968:0:231961:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576179.343970:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343973:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576179.343975:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343976:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576179.343979:0:231961:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576179.343980:0:231961:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788576179.343982:0:231961:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576179.343984:0:231961:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788576179.343987:0:231961:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788576179.343989:0:231961:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788576179.343992:0:231961:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.343994:0:231961:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576179.344006:0:231961:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576179.344008:0:231961:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576179.344010:0:231961:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788576179.344013:0:231961:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576179.344016:0:231961:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576179.344017:0:231961:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576179.344020:0:231961:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576179.344023:0:231961:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.344025:0:231961:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576179.344027:0:231961:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576179.344029:0:231961:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.344030:0:231961:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576179.344035:0:231961:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788576179.344037:0:231961:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576179.344039:0:231961:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576179.344040:0:231961:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576179.344042:0:231961:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576179.344057:0:231961:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576179.344059:0:231961:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576179.344064:0:231961:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9e000. 00000100:00000001:0.0:1788576179.344068:0:231961:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576179.344073:0:231961:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516915584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576179.344081:0:231961:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576179.344083:0:231961:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576179.344087:0:231961:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516915584/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576179.344094:0:231961:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576179.344101:0:231961:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16876) 00000100:00000001:0.0:1788576179.842763:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576179.842766:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576179.842768:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576179.842797:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576179.842799:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576179.842801:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576179.842803:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576179.842805:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576179.842844:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576180.545241:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576180.545245:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576180.545248:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576180.865084:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576180.865095:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576180.865097:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576180.865105:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576180.865110:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576180.865115:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576180.865119:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576180.865121:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576180.865124:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576180.865428:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576180.865432:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576180.865433:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576180.865435:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576180.865438:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576180.865440:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576180.865442:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576180.865443:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576180.865445:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576181.890272:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576181.890277:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576181.890279:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576181.890284:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576181.890287:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576181.890291:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576181.890294:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576181.890295:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576181.890298:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576182.593705:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576182.593711:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576182.593729:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576182.912929:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576182.912940:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576182.912943:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576182.912950:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576182.912955:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576182.912960:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576182.912965:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576182.912969:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576182.912978:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576182.913258:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576182.913261:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576182.913262:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576182.913265:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576182.913267:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576182.913269:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576182.913271:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576182.913272:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576182.913274:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576183.168920:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311543027, at 4311543040 00000100:00000010:0.0:1788576183.168938:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc4ac9ab2a0. 00000100:00000001:0.0:1788576183.936906:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576183.936913:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576183.936915:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576183.936919:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576183.936923:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576183.936926:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576183.936929:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576183.936931:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576183.936935:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576183.937649:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576183.937652:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576183.937654:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576183.937657:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576183.937659:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576183.937661:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576183.937663:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576183.937682:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576183.937685:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1788576184.449146:0:2390:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8fc4880929c0] -> 12345-192.168.202.28@tcp (6) 00000800:00000200:0.0:1788576184.449158:0:2390:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.28@tcp ip 192.168.202.28:988 00000800:00000200:0.0:1788576184.449163:0:2390:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8fc4858bcf00 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:0.0:1788576184.960907:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576184.960915:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576184.960918:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576184.960923:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576184.960927:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576184.960931:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576184.960934:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576184.960936:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576184.960939:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576184.962243:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576184.962247:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576184.962248:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576184.962251:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576184.962254:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576184.962256:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576184.962258:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576184.962259:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576184.962261:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576185.665194:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576185.665198:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576185.665201:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576185.984926:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576185.984935:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576185.984938:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576185.984943:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576185.984947:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576185.984952:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576185.984954:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576185.984957:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576185.984960:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576185.985039:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576185.985044:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576185.985063:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576185.985066:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576185.985069:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576185.985071:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576185.985072:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576185.985074:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576185.985076:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576185.985313:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576185.985318:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576185.985321:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576185.985327:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576185.985332:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576185.985337:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576185.985349:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576185.985352:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576185.985354:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576187.009101:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576187.009109:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576187.009111:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576187.009116:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576187.009119:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576187.009308:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576187.009311:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576187.009314:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576187.009318:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576187.713170:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576187.713173:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576187.713176:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576188.033604:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576188.033609:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576188.033612:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576188.033617:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576188.033625:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576188.033630:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.033637:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576188.033641:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.033645:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576188.034223:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576188.034243:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576188.034259:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576188.034262:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576188.034265:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576188.034267:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.034269:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576188.034270:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576188.034272:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576188.289984:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311548142, at 4311548161 00000100:00000010:0.0:1788576188.289995:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674960. 00000100:00000001:0.0:1788576189.057257:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576189.057262:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576189.057263:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576189.057267:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576189.057270:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576189.057272:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576189.057274:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576189.057276:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576189.057282:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576189.572928:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3d5a86eb-f614-41e3-98bd-af00951045c6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576189.572940:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788576189.572943:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576189.572960:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4acae6800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576189.572964:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576189.572972:0:265864:0:(import.c:545:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 16877 00000400:00000200:0.0:1788576189.572981:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576189.572987:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576189.572990:0:265864:0:(import.c:630:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576189.572993:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.572996:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576189.572998:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573001:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576189.573004:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576189.573024:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852407 00000020:00000001:0.0:1788576189.573027:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576189.573029:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc490e71800 refcount=3 00000020:00000001:0.0:1788576189.573031:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000020:00000001:0.0:1788576189.573034:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673140267008 : -123400569284608 : ffff8fc490e71800) 00000100:00000040:0.0:1788576189.573037:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573039:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576189.573040:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573042:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576189.573045:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc490e71800 : new refcount 2 00000100:00080000:0.0:1788576189.573048:0:265864:0:(import.c:665:import_select_connection()) MGC192.168.202.128@tcp: import ffff8fc4acae6800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573051:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576189.573054:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576189.573057:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576189.573060:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573062:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1788576189.573064:0:265864:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576189.573070:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1c00. 00000020:00000040:0.0:1788576189.573073:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4acae6800 refcount=4 obd=MGC192.168.202.128@tcp 02000000:00000001:0.0:1788576189.573077:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576189.573078:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573081:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576189.573083:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576189.573087:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576189.573097:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c000. 00000100:00000001:0.0:1788576189.573105:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576189.573114:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1c00 x1875440516916352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576189.573124:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576189.573130:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576189.573134:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ac03888] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1c00 x1875440516916352/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576189.573159:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576189.573162:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576189.573165:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788576189.573166:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576189.573167:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc485ade000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576189.573170:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576189.573172:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16877 00000400:00000200:0.0:1788576189.573176:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576189.573179:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576189.573181:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576189.573184:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573185:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576189.573187:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573188:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576189.573190:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576189.573191:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf1285245b 00000020:00000001:0.0:1788576189.573193:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576189.573195:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88800 refcount=5 00000020:00000001:0.0:1788576189.573197:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000020:00000001:0.0:1788576189.573199:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506314240 : -123400203237376 : ffff8fc4a6b88800) 00000100:00000040:0.0:1788576189.573202:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573204:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576189.573206:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573208:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576189.573210:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88800 : new refcount 4 00000100:00080000:0.0:1788576189.573212:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffff8fc490e72800: import ffff8fc485ade000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573214:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576189.573222:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576189.573239:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576189.573241:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573243:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576189.573248:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576189.573250:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573252:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576189.573255:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2680. 00000020:00000040:0.0:1788576189.573258:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc485ade000 refcount=6 obd=lustre-MDT0000-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576189.573260:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576189.573261:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573262:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576189.573264:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576189.573265:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576189.573269:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9f000. 00000100:00000001:0.0:1788576189.573272:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576189.573276:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2680 x1875440516916480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576189.573282:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576189.573284:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576189.573288:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1c25293] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2680 x1875440516916480/t0(0) o38->lustre-MDT0000-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576189.573295:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576189.573297:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576189.573300:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788576189.573302:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576189.573304:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc486471000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576189.573306:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576189.573309:0:265864:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16877 00000400:00000200:0.0:1788576189.573313:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576189.573317:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576189.573319:0:265864:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576189.573329:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573331:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576189.573333:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573335:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576189.573337:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576189.573338:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852462 00000020:00000001:0.0:1788576189.573339:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576189.573341:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8a000 refcount=5 00000020:00000001:0.0:1788576189.573343:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000020:00000001:0.0:1788576189.573345:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506320384 : -123400203231232 : ffff8fc4a6b8a000) 00000100:00000040:0.0:1788576189.573349:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573350:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576189.573352:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573354:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576189.573356:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8a000 : new refcount 4 00000100:00080000:0.0:1788576189.573357:0:265864:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffff8fc490e72800: import ffff8fc486471000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573359:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576189.573361:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576189.573364:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576189.573366:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573368:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576189.573370:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683395 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788576189.573373:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573374:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576189.573378:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c1f80. 00000020:00000040:0.0:1788576189.573380:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc486471000 refcount=10 obd=lustre-MDT0001-mdc-ffff8fc490e72800 02000000:00000001:0.0:1788576189.573382:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576189.573384:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573385:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576189.573386:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576189.573388:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576189.573392:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9cc00. 00000100:00000001:0.0:1788576189.573400:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576189.573405:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c1f80 x1875440516916608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576189.573412:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576189.573414:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576189.573417:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008fabd9e9] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c1f80 x1875440516916608/t0(0) o38->lustre-MDT0001-mdc-ffff8fc490e72800@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576189.573425:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576189.573427:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576189.573430:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788576189.573432:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576189.573433:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576189.573436:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576189.573438:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16877 00000400:00000200:0.0:1788576189.573442:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576189.573445:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576189.573447:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576189.573450:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573452:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576189.573454:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573456:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576189.573458:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576189.573459:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852470 00000020:00000001:0.0:1788576189.573461:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576189.573463:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b8e800 refcount=4 00000020:00000001:0.0:1788576189.573465:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000020:00000001:0.0:1788576189.573467:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506338816 : -123400203212800 : ffff8fc4a6b8e800) 00000100:00000040:0.0:1788576189.573470:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573471:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576189.573478:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573480:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576189.573482:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b8e800 : new refcount 3 00000100:00080000:0.0:1788576189.573484:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffff8fc490e72800: import ffff8fc4912d1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573487:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576189.573488:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576189.573491:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576189.573493:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573494:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576189.573496:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576189.573499:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573500:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576189.573504:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c2300. 00000020:00000040:0.0:1788576189.573506:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1000 refcount=6 obd=lustre-OST0000-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576189.573508:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576189.573509:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573510:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576189.573512:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576189.573514:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576189.573517:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c800. 00000100:00000001:0.0:1788576189.573521:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576189.573526:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c2300 x1875440516916736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576189.573532:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576189.573534:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576189.573537:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e27a4d6] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c2300 x1875440516916736/t0(0) o8->lustre-OST0000-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576189.573544:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788576189.573547:0:265864:0:(pinger.c:229:ptlrpc_pinger_process_import()) 51d6aedc-ea7c-426a-b495-22e82c13ee75->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788576189.573550:0:265864:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788576189.573551:0:265864:0:(import.c:745:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788576189.573557:0:265864:0:(import.c:65:import_set_state_nolock()) ffff8fc4912d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788576189.573560:0:265864:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788576189.573562:0:265864:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: connect to NID 192.168.202.128@tcp last attempt 16877 00000400:00000200:0.0:1788576189.573566:0:265864:0:(api-ni.c:1618:lnet_nid4_cpt_hash()) Match nid 192.168.202.128@tcp to cpt 0 00000400:00000200:0.0:1788576189.573569:0:265864:0:(peer.c:1689:LNetPeerDiscovered()) Peer NID 192.168.202.128@tcp is discovered: rc = 1 00000100:00080000:0.0:1788576189.573571:0:265864:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788576189.573574:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573576:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576189.573578:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573580:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000001:0.0:1788576189.573582:0:265864:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1788576189.573583:0:265864:0:(genops.c:804:class_conn2export()) looking for export cookie 0x7d52aadf12852477 00000020:00000001:0.0:1788576189.573584:0:265864:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1788576189.573587:0:265864:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8fc4a6b88000 refcount=4 00000020:00000001:0.0:1788576189.573589:0:265864:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000020:00000001:0.0:1788576189.573591:0:265864:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446620673506312192 : -123400203239424 : ffff8fc4a6b88000) 00000100:00000040:0.0:1788576189.573594:0:265864:0:(lustre_net.h:1984:ptlrpc_connection_put()) PUT conn=ffff8fc4884f8dc0 refcount 14 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573595:0:265864:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788576189.573597:0:265864:0:(connection.c:162:ptlrpc_connection_addref()) conn=ffff8fc4884f8dc0 refcount 15 to 192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573598:0:265864:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446620672996117952 : -123400713433664 : ffff8fc4884f8dc0) 00000020:00000040:0.0:1788576189.573600:0:265864:0:(genops.c:879:class_export_put()) PUTting export ffff8fc4a6b88000 : new refcount 3 00000100:00080000:0.0:1788576189.573603:0:265864:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffff8fc490e72800: import ffff8fc4912d1800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00000001:0.0:1788576189.573606:0:265864:0:(import.c:672:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788576189.573607:0:265864:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788576189.573610:0:265864:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788576189.573612:0:265864:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573614:0:265864:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1788576189.573615:0:265864:0:(osc_request.c:3889:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683395 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788576189.573617:0:265864:0:(osc_request.c:3892:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573619:0:265864:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788576189.573638:0:265864:0:(client.c:613:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8fc4b80c0700. 00000020:00000040:0.0:1788576189.573641:0:265864:0:(genops.c:1210:class_import_get()) import ffff8fc4912d1800 refcount=7 obd=lustre-OST0001-osc-ffff8fc490e72800 02000000:00000001:0.0:1788576189.573644:0:265864:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788576189.573645:0:265864:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576189.573647:0:265864:0:(client.c:865:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788576189.573648:0:265864:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788576189.573650:0:265864:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788576189.573654:0:265864:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8fc4a6c9c400. 00000100:00000001:0.0:1788576189.573658:0:265864:0:(client.c:945:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576189.573672:0:265864:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8fc4b80c0700 x1875440516916864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788576189.573679:0:265864:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788576189.573681:0:265864:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576189.573685:0:265864:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e90af13d] to pc [ptlrpcd_rcv+-1] req@ffff8fc4b80c0700 x1875440516916864/t0(0) o8->lustre-OST0001-osc-ffff8fc490e72800@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788576189.573692:0:265864:0:(import.c:877:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576189.573697:0:265864:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (16887) 00000020:00000001:0.0:1788576190.786288:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576190.786292:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576190.786296:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576191.105506:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576191.105512:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576191.105516:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576191.105521:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576191.105535:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576191.105538:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576191.105540:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576191.105542:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576191.105544:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576192.128996:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576192.129006:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576192.129009:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576192.129017:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576192.129021:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576192.129027:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576192.129055:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576192.129059:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576192.129064:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576192.131445:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576192.131456:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576192.131459:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576192.131465:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576192.131469:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576192.131474:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576192.131477:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576192.131480:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576192.131482:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576192.833329:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576192.833338:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576192.833343:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1788576193.414274:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311553266, at 4311553285 00000100:00000010:0.0:1788576193.414290:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674e40. 00000100:00000001:0.0:1788576194.177669:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576194.177672:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576194.177673:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576194.177676:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576194.177678:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576194.177680:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576194.177682:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576194.177683:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576194.177686:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576195.201292:0:265054:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576195.201299:0:265054:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576195.201301:0:265054:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576195.201306:0:265054:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4a048a540 going to sleep for 0 seconds 00000100:00000001:0.0:1788576195.201310:0:265054:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576195.201312:0:265054:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576195.201315:0:265054:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576195.201317:0:265054:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576195.201320:0:265054:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576195.201603:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576195.201607:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576195.201626:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576195.201628:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576195.201631:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576195.201632:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576195.201634:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576195.201635:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576195.201638:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576195.202185:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576195.202188:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576195.202189:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576195.202191:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576195.202193:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576195.202195:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576195.202197:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576195.202198:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576195.202199:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576195.908538:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576195.908547:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576195.908552:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576196.225290:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576196.225293:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576196.225294:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576196.225297:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576196.225299:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576196.225300:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.225302:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576196.225303:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576196.225306:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576196.929443:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576196.929446:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576196.929450:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576197.253313:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576197.253321:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576197.253323:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576197.253329:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576197.253332:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576197.253336:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576197.253339:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576197.253356:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576197.253360:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576197.953249:0:251537:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576197.953253:0:251537:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576197.953256:0:251537:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576198.272938:0:265053:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576198.272947:0:265053:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576198.272949:0:265053:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576198.272955:0:265053:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5300 going to sleep for 0 seconds 00000100:00000001:0.0:1788576198.272958:0:265053:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576198.272961:0:265053:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576198.272963:0:265053:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576198.272966:0:265053:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576198.272969:0:265053:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576198.273153:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576198.273157:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576198.273158:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576198.273161:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576198.273163:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576198.273165:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576198.273167:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576198.273168:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576198.273170:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788576198.529235:0:266056:0:(connection.c:75:cpu_latency_work()) work item of ffff8fc4a21a7000 (cpu 0) has reached its deadline 4311558383, at 4311558400 00000100:00000010:0.0:1788576198.529252:0:266056:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8fc491674d20. 00000100:00000001:0.0:1788576199.297377:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576199.297381:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576199.297382:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576199.297385:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576199.297388:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576199.297390:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576199.297391:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576199.297393:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576199.297395:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576200.321104:0:265051:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576200.321108:0:265051:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576200.321109:0:265051:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576200.321161:0:265051:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5000 going to sleep for 0 seconds 00000100:00000001:0.0:1788576200.321164:0:265051:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576200.321166:0:265051:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576200.321168:0:265051:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576200.321170:0:265051:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576200.321172:0:265051:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576200.321384:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576200.321389:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576200.321390:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576200.321393:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576200.321395:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576200.321397:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576200.321399:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576200.321400:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576200.321409:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576201.025439:0:235758:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576201.025443:0:235758:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576201.025447:0:235758:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788576201.345309:0:265052:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576201.345312:0:265052:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576201.345314:0:265052:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576201.345317:0:265052:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b8075780 going to sleep for 0 seconds 00000100:00000001:0.0:1788576201.345319:0:265052:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576201.345321:0:265052:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.345324:0:265052:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576201.345325:0:265052:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.345327:0:265052:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788576201.345973:0:265055:0:(client.c:2575:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788576201.345975:0:265055:0:(client.c:2647:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788576201.345977:0:265055:0:(client.c:2675:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788576201.345979:0:265055:0:(client.c:2723:ptlrpc_set_wait()) set ffff8fc4b82f5a80 going to sleep for 0 seconds 00000100:00000001:0.0:1788576201.345982:0:265055:0:(client.c:1998:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788576201.345984:0:265055:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.345986:0:265055:0:(client.c:1384:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788576201.345987:0:265055:0:(client.c:1443:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788576201.345989:0:265055:0:(client.c:2473:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788576202.049147:0:247415:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788576202.049152:0:247415:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788576202.049201:0:247415:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 165514 lines, 165514 kept, 0 dropped, 0 bad.